feat: Implement passive XHR/fetch capture using Playwright
- Added `capture_passive_playwright.py` for synchronous passive capture of XHR/fetch responses. - Added `capture_passive_playwright_async.py` for asynchronous passive capture using Playwright's async API. - Introduced `save_page_snapshots.py` to save full page HTML snapshots with deduplication. - Removed obsolete CSRF token capture JSON file.
This commit is contained in:
Ross
+2
-1
@@ -69,4 +69,5 @@ secrets.json
|
||||
playwright_profile
|
||||
|
||||
captured/
|
||||
xhr_captured/
|
||||
xhr_captured/
|
||||
xhr_captured_async/
|
||||
@@ -0,0 +1,459 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Passive XHR/fetch capture using Playwright.
|
||||
|
||||
This script launches a persistent browser profile (so you can log in),
|
||||
optionally attempts auto-login, then attaches a response handler and saves
|
||||
every XHR/fetch response body and metadata into `--output-dir` while you
|
||||
browse. It is intentionally passive: it doesn't reload the page unless you
|
||||
ask it to; it simply listens and saves responses as they happen.
|
||||
|
||||
Usage:
|
||||
python scrapers/capture_passive_playwright.py --output-dir xhr_captured --continuous
|
||||
|
||||
Options:
|
||||
--username/--password or STATDX_USERNAME/STATDX_PASSWORD via .env for auto-login
|
||||
--no-prompt to start listening immediately
|
||||
--capture-types to adjust which resource types to record (default: xhr,fetch)
|
||||
"""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import mimetypes
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
import asyncio
|
||||
try:
|
||||
from playwright._impl._errors import TargetClosedError as _PlaywrightTargetClosedError
|
||||
TARGET_CLOSED_EXCEPTIONS = (_PlaywrightTargetClosedError,)
|
||||
except Exception:
|
||||
# If we cannot import Playwright internal error types, fall back to Exception
|
||||
TARGET_CLOSED_EXCEPTIONS = (Exception,)
|
||||
from dotenv import load_dotenv
|
||||
|
||||
logger.remove()
|
||||
logger.add("capture_passive_playwright.log", rotation="10 MB")
|
||||
logger.add(sys.stderr, level="TRACE")
|
||||
# Keep detailed logs in the file only to avoid noisy console output while you browse.
|
||||
# We will print a short summary to stdout when the browser/context is closed.
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
def sanitize(s: str) -> str:
|
||||
out = []
|
||||
for ch in s:
|
||||
if ch.isalnum() or ch in ('-', '_', '.'):
|
||||
out.append(ch)
|
||||
else:
|
||||
out.append('_')
|
||||
name = ''.join(out)
|
||||
if len(name) > 180:
|
||||
name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
|
||||
return name
|
||||
|
||||
|
||||
def guess_ext(ctype: str) -> str:
|
||||
if not ctype:
|
||||
return 'bin'
|
||||
mt = ctype.split(';')[0].strip()
|
||||
ext = mimetypes.guess_extension(mt)
|
||||
if ext:
|
||||
return ext.lstrip('.')
|
||||
if mt.startswith('image/'):
|
||||
return mt.split('/')[1]
|
||||
if mt == 'application/json':
|
||||
return 'json'
|
||||
if mt == 'text/html':
|
||||
return 'html'
|
||||
if mt.startswith('text/'):
|
||||
return 'txt'
|
||||
return 'bin'
|
||||
|
||||
|
||||
def now_ts():
|
||||
return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
|
||||
def save_binary(path: Path, data: bytes):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
def save_text(path: Path, text: str):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def try_pretty_json(s: str):
|
||||
try:
|
||||
parsed = json.loads(s)
|
||||
return json.dumps(parsed, ensure_ascii=False, indent=2)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def attempt_autologin(page, username, password, post_login_selector, wait_after=3.0):
|
||||
if not username or not password:
|
||||
return False
|
||||
logger.info('Attempting auto-login')
|
||||
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]']
|
||||
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
||||
|
||||
user_sel = None
|
||||
for sel in user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
pass_sel = None
|
||||
for sel in pass_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
pass_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel or not pass_sel:
|
||||
logger.warning('Could not find login fields for auto-login')
|
||||
return False
|
||||
|
||||
try:
|
||||
page.fill(user_sel, username)
|
||||
time.sleep(0.1)
|
||||
page.fill(pass_sel, password)
|
||||
time.sleep(0.1)
|
||||
# try to submit
|
||||
submit_try = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")']
|
||||
clicked = False
|
||||
for s in submit_try:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if not clicked:
|
||||
page.press(pass_sel, 'Enter')
|
||||
|
||||
if post_login_selector:
|
||||
try:
|
||||
page.wait_for_selector(post_login_selector, timeout=10000)
|
||||
except Exception:
|
||||
logger.info('Post-login selector not found in timeout')
|
||||
|
||||
time.sleep(wait_after)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception('Auto-login failed')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--url', '-u', default='https://app.statdx.com/')
|
||||
parser.add_argument('--profile', '-p', default='playwright_profile')
|
||||
parser.add_argument('--output-dir', '-o', default='xhr_captured')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
parser.add_argument('--continuous', action='store_true', help='Keep listening and saving as you browse')
|
||||
parser.add_argument('--no-prompt', action='store_true', help='Start listening immediately without waiting for Enter')
|
||||
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
|
||||
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
|
||||
parser.add_argument('--post-login-selector', default='#ds-app')
|
||||
parser.add_argument('--capture-types', default='xhr,fetch,document,other', help='Comma-separated resource types to capture (empty = all)')
|
||||
args = parser.parse_args()
|
||||
|
||||
output = Path(args.output_dir)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
index_path = output / 'capture_index.jsonl'
|
||||
|
||||
username = args.username or os.getenv('STATDX_USERNAME')
|
||||
password = args.password or os.getenv('STATDX_PASSWORD')
|
||||
capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
|
||||
|
||||
with sync_playwright() as p:
|
||||
# launch persistent context
|
||||
try:
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome')
|
||||
except Exception:
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
page = context.new_page()
|
||||
# NOTE: don't navigate until we've attached listeners so we don't miss
|
||||
# early requests from the initial load.
|
||||
|
||||
# response handler (define before we attach listeners)
|
||||
def on_response(resp):
|
||||
"""Safe response handler: reads response bodies but tolerates browser/context close.
|
||||
|
||||
Playwright can raise TargetClosedError or asyncio.CancelledError when the
|
||||
browser or context closes while we're trying to read a body. Those are
|
||||
normal during shutdown and should not produce noisy tracebacks. This
|
||||
handler therefore catches those and returns quietly.
|
||||
"""
|
||||
logger.debug(f'Response: {getattr(resp, "status", "?")} {getattr(resp, "url", "?")}')
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = (req.resource_type or '').lower()
|
||||
# Log every response minimally so user sees activity
|
||||
logger.info('Response event: %s %s %s', getattr(resp, 'status', '?'), rtype, req.url)
|
||||
|
||||
# write debug line to disk so we can verify events even if console is quiet
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\t{getattr(resp, 'status', '?')}\t{rtype}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# If capture_types is empty, capture everything. Otherwise check membership.
|
||||
allow = False
|
||||
if not capture_types:
|
||||
allow = True
|
||||
elif rtype in capture_types:
|
||||
allow = True
|
||||
# Heuristic: always capture document content endpoints we care about
|
||||
if '/document/content/' in req.url or '/document/summary/' in req.url:
|
||||
allow = True
|
||||
if not allow:
|
||||
return
|
||||
|
||||
url = req.url
|
||||
ts = now_ts()
|
||||
status = getattr(resp, 'status', None)
|
||||
req_headers = dict(req.headers)
|
||||
resp_headers = dict(resp.headers)
|
||||
|
||||
meta = {
|
||||
'url': url,
|
||||
'method': req.method,
|
||||
'resource_type': rtype,
|
||||
'status': status,
|
||||
'timestamp': ts,
|
||||
'request_headers': req_headers,
|
||||
'response_headers': resp_headers,
|
||||
}
|
||||
|
||||
# request post data if any
|
||||
try:
|
||||
post = req.post_data
|
||||
meta['request_post_data'] = post
|
||||
except Exception:
|
||||
meta['request_post_data'] = None
|
||||
|
||||
ctype = (resp_headers.get('content-type') or '').lower()
|
||||
safe = sanitize(url.replace('https://', '').replace('http://', ''))
|
||||
h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8]
|
||||
|
||||
body_path = None
|
||||
excerpt = None
|
||||
# try textual
|
||||
try:
|
||||
if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
|
||||
try:
|
||||
txt = resp.text()
|
||||
except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
|
||||
logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
|
||||
return
|
||||
# pretty-print JSON if possible
|
||||
pretty = try_pretty_json(txt)
|
||||
if pretty is not None:
|
||||
fname = f"{safe}_{h}_{ts}.json"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, pretty)
|
||||
excerpt = pretty[:800]
|
||||
else:
|
||||
fname = f"{safe}_{h}_{ts}.txt"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, txt)
|
||||
excerpt = txt[:800]
|
||||
else:
|
||||
try:
|
||||
data = resp.body()
|
||||
except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
|
||||
logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
|
||||
return
|
||||
ext = guess_ext(ctype)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(output / fname)
|
||||
save_binary(output / fname, data)
|
||||
except Exception as e:
|
||||
try:
|
||||
# fallback to binary
|
||||
try:
|
||||
data = resp.body()
|
||||
except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,):
|
||||
logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '<unknown>'))
|
||||
return
|
||||
ext = guess_ext(ctype)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(output / fname)
|
||||
save_binary(output / fname, data)
|
||||
except Exception:
|
||||
logger.debug('Failed to save body for %s: %s', url, e)
|
||||
|
||||
meta['response_body_file'] = body_path
|
||||
meta['response_excerpt'] = excerpt
|
||||
|
||||
meta_name = f"{safe}_{h}_{ts}.json"
|
||||
meta_path = output / meta_name
|
||||
save_text(meta_path, json.dumps(meta, ensure_ascii=False, indent=2))
|
||||
|
||||
# append index
|
||||
try:
|
||||
with open(index_path, 'a', encoding='utf-8') as idx:
|
||||
idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(meta_path), 'excerpt': excerpt}, ensure_ascii=False) + '\n')
|
||||
except Exception:
|
||||
logger.exception('Failed to write index entry')
|
||||
|
||||
logger.info('Captured %s %s -> %s', rtype, url, meta_path.name)
|
||||
except Exception as e:
|
||||
# suppress noisy Playwright shutdown traces
|
||||
if isinstance(e, TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,)) or \
|
||||
(hasattr(e, 'args') and any('Target page, context or browser has been closed' in str(a) for a in e.args)):
|
||||
logger.debug('Response handler aborted: target closed')
|
||||
return
|
||||
logger.exception('Error in response handler')
|
||||
|
||||
# helper to attach listeners to a page
|
||||
def attach_listeners_to_page(pg):
|
||||
try:
|
||||
pg.on('response', on_response)
|
||||
# log requests and finished requests for debug
|
||||
def on_request(req):
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def on_request_finished(req):
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
pg.on('request', on_request)
|
||||
pg.on('requestfinished', on_request_finished)
|
||||
# also log request failures for visibility
|
||||
def on_req_failed(req):
|
||||
try:
|
||||
logger.warning('Request failed: %s %s', req.failure, req.url)
|
||||
except Exception:
|
||||
pass
|
||||
pg.on('requestfailed', on_req_failed)
|
||||
logger.debug('Attached listeners to page: %s', pg.url)
|
||||
except Exception:
|
||||
logger.exception('Failed to attach listeners to page')
|
||||
|
||||
try:
|
||||
# navigation and lifecycle events
|
||||
def on_navigate():
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tNAVIGATE\t{pg.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
pg.on('load', lambda: on_navigate())
|
||||
pg.on('domcontentloaded', lambda: on_navigate())
|
||||
pg.on('framenavigated', lambda frame: on_navigate())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# attach to any existing pages in the context
|
||||
try:
|
||||
for pg in context.pages:
|
||||
attach_listeners_to_page(pg)
|
||||
except Exception:
|
||||
logger.debug('No existing pages to attach')
|
||||
|
||||
# attach to future pages
|
||||
try:
|
||||
context.on('page', lambda new_page: attach_listeners_to_page(new_page))
|
||||
except Exception:
|
||||
logger.debug('Could not attach context.page listener')
|
||||
|
||||
# also attach at context level to catch responses not tied to a single page
|
||||
try:
|
||||
context.on('response', lambda resp: on_response(resp))
|
||||
except Exception:
|
||||
logger.debug('Context-level response listener not available')
|
||||
|
||||
# add context-level request listeners too (some fetches/service-workers
|
||||
# may appear at the context level rather than page)
|
||||
try:
|
||||
def _ctx_request(req):
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
def _ctx_req_finished(req):
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tCTX_REQUEST_FINISHED\t{req.method}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
context.on('request', lambda r: _ctx_request(r))
|
||||
context.on('requestfinished', lambda r: _ctx_req_finished(r))
|
||||
except Exception:
|
||||
logger.debug('Context-level request listeners not available')
|
||||
|
||||
# try auto-login
|
||||
if username and password:
|
||||
attempt_autologin(page, username, password, args.post_login_selector)
|
||||
|
||||
# navigate after listeners attached so we capture initial load
|
||||
try:
|
||||
logger.info('Opening %s', args.url)
|
||||
page.goto(args.url)
|
||||
except Exception:
|
||||
logger.debug('Initial navigation failed or was skipped')
|
||||
|
||||
# If interactive and not no-prompt, let user finish manual login/navigation
|
||||
if not args.no_prompt:
|
||||
try:
|
||||
input('When you have logged in and are ready, press Enter to start passive capture...\n')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info(f'Starting passive response capture (types: {", ".join(sorted(capture_types))})')
|
||||
|
||||
# attach the response handler to the initial page
|
||||
page.on('response', on_response)
|
||||
|
||||
logger.info('Passive listener attached. Continue browsing in the opened browser window.')
|
||||
|
||||
if args.continuous:
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logger.info('Interrupted by user; finishing and closing browser')
|
||||
else:
|
||||
# one-shot: wait a short while to gather responses then exit
|
||||
time.sleep(5)
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,307 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Async passive XHR/fetch capture using Playwright's async API.
|
||||
|
||||
This script mirrors the sync version but uses async/await to avoid blocking
|
||||
the event loop. It attaches listeners before navigating and prints a small
|
||||
heartbeat that shows how many responses we've captured so you can see
|
||||
activity live.
|
||||
|
||||
Usage:
|
||||
python scrapers/capture_passive_playwright_async.py --output-dir xhr_captured_async --continuous
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import mimetypes
|
||||
import hashlib
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except Exception:
|
||||
raise
|
||||
|
||||
|
||||
def sanitize(s: str) -> str:
|
||||
out = []
|
||||
for ch in s:
|
||||
if ch.isalnum() or ch in ('-', '_', '.'):
|
||||
out.append(ch)
|
||||
else:
|
||||
out.append('_')
|
||||
name = ''.join(out)
|
||||
if len(name) > 180:
|
||||
name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
|
||||
return name
|
||||
|
||||
|
||||
def guess_ext(ctype: str) -> str:
|
||||
if not ctype:
|
||||
return 'bin'
|
||||
mt = ctype.split(';')[0].strip()
|
||||
ext = mimetypes.guess_extension(mt)
|
||||
if ext:
|
||||
return ext.lstrip('.')
|
||||
if mt.startswith('image/'):
|
||||
return mt.split('/')[1]
|
||||
if mt == 'application/json':
|
||||
return 'json'
|
||||
if mt == 'text/html':
|
||||
return 'html'
|
||||
if mt.startswith('text/'):
|
||||
return 'txt'
|
||||
return 'bin'
|
||||
|
||||
|
||||
def now_ts():
|
||||
return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
|
||||
def save_text(path: Path, text: str):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def save_binary(path: Path, data: bytes):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'wb') as f:
|
||||
f.write(data)
|
||||
|
||||
|
||||
async def run(args):
|
||||
output = Path(args.output_dir)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
index_path = output / 'capture_index.jsonl'
|
||||
|
||||
capture_count = 0
|
||||
capture_count_lock = asyncio.Lock()
|
||||
|
||||
async def heartbeat():
|
||||
while True:
|
||||
await asyncio.sleep(5)
|
||||
async with capture_count_lock:
|
||||
print(f"[heartbeat] captured={capture_count}")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser_type = p.chromium
|
||||
# Launch persistent context
|
||||
if args.channel:
|
||||
try:
|
||||
context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
|
||||
except Exception:
|
||||
print(f'Launch with channel {args.channel} failed; falling back to bundled Chromium')
|
||||
context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
else:
|
||||
context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
page = await context.new_page()
|
||||
|
||||
async def on_response(resp):
|
||||
nonlocal capture_count
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = (req.resource_type or '').lower()
|
||||
|
||||
# write quick debug line
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tRESP\t{resp.status}\t{rtype}\t{req.url}\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
allow = False
|
||||
if not args.capture_types:
|
||||
allow = True
|
||||
elif rtype in args.capture_types:
|
||||
allow = True
|
||||
if '/document/content/' in req.url or '/document/summary/' in req.url:
|
||||
allow = True
|
||||
if not allow:
|
||||
return
|
||||
|
||||
url = req.url
|
||||
ts = now_ts()
|
||||
resp_headers = dict(resp.headers)
|
||||
ctype = (resp_headers.get('content-type') or '').lower()
|
||||
|
||||
safe = sanitize(url.replace('https://', '').replace('http://', ''))
|
||||
h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8]
|
||||
|
||||
body_path = None
|
||||
excerpt = None
|
||||
|
||||
try:
|
||||
if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
|
||||
try:
|
||||
txt = await resp.text()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
if 'Target page, context or browser has been closed' in str(e):
|
||||
return
|
||||
raise
|
||||
# try pretty JSON
|
||||
try:
|
||||
parsed = json.loads(txt)
|
||||
pretty = json.dumps(parsed, ensure_ascii=False, indent=2)
|
||||
fname = f"{safe}_{h}_{ts}.json"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, pretty)
|
||||
excerpt = pretty[:800]
|
||||
except Exception:
|
||||
fname = f"{safe}_{h}_{ts}.txt"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, txt)
|
||||
excerpt = txt[:800]
|
||||
else:
|
||||
try:
|
||||
data = await resp.body()
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
except Exception as e:
|
||||
if 'Target page, context or browser has been closed' in str(e):
|
||||
return
|
||||
raise
|
||||
ext = guess_ext(ctype)
|
||||
# Put images into an images/ subdirectory, other binaries into assets/
|
||||
if ctype.startswith('image/'):
|
||||
subdir = output / 'images'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
else:
|
||||
subdir = output / 'assets'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
except Exception as e:
|
||||
# final fallback: try binary
|
||||
try:
|
||||
data = await resp.body()
|
||||
ext = guess_ext(ctype)
|
||||
if ctype.startswith('image/'):
|
||||
subdir = output / 'images'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
else:
|
||||
subdir = output / 'assets'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
except Exception:
|
||||
# skip if still failing
|
||||
return
|
||||
|
||||
meta = {
|
||||
'url': url,
|
||||
'resource_type': rtype,
|
||||
'status': resp.status,
|
||||
'timestamp': ts,
|
||||
'response_headers': resp_headers,
|
||||
'response_body_file': body_path,
|
||||
'response_excerpt': excerpt,
|
||||
}
|
||||
meta_name = f"{safe}_{h}_{ts}.meta.json"
|
||||
save_text(output / meta_name, json.dumps(meta, ensure_ascii=False, indent=2))
|
||||
try:
|
||||
with open(index_path, 'a', encoding='utf-8') as idx:
|
||||
idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt}, ensure_ascii=False) + '\n')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
async with capture_count_lock:
|
||||
capture_count += 1
|
||||
|
||||
except Exception:
|
||||
# swallow unexpected exceptions from handler so event loop stays healthy
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tRESP_HANDLER_ERROR\n")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# helpers to attach to page
|
||||
def attach_page_listeners(pg):
|
||||
pg.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||||
pg.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n"))
|
||||
pg.on('requestfinished', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n"))
|
||||
|
||||
# attach listeners to existing pages
|
||||
for pg in context.pages:
|
||||
attach_page_listeners(pg)
|
||||
|
||||
# attach to future pages
|
||||
context.on('page', lambda new_page: attach_page_listeners(new_page))
|
||||
|
||||
# context-level response and request listeners
|
||||
context.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||||
context.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n"))
|
||||
|
||||
# Start heartbeat
|
||||
hb_task = asyncio.create_task(heartbeat())
|
||||
|
||||
# Navigate after listeners attached
|
||||
try:
|
||||
await page.goto(args.url)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# If interactive, let user log in; otherwise start capture immediately
|
||||
if not args.no_prompt:
|
||||
print('When you have logged in in the opened browser, press Enter here to continue and capture...')
|
||||
try:
|
||||
await asyncio.get_event_loop().run_in_executor(None, input)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print(f'Starting async passive capture (types: {args.capture_types})')
|
||||
|
||||
try:
|
||||
if args.continuous:
|
||||
# keep running until KeyboardInterrupt
|
||||
while True:
|
||||
await asyncio.sleep(1)
|
||||
else:
|
||||
await asyncio.sleep(5)
|
||||
except KeyboardInterrupt:
|
||||
print('Interrupted; closing...')
|
||||
finally:
|
||||
hb_task.cancel()
|
||||
try:
|
||||
await context.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--url', default='https://app.statdx.com/')
|
||||
parser.add_argument('--profile', default='playwright_profile')
|
||||
parser.add_argument('--output-dir', default='xhr_captured_async')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
parser.add_argument('--continuous', action='store_true')
|
||||
parser.add_argument('--no-prompt', action='store_true')
|
||||
parser.add_argument('--channel', default=os.getenv('PLAYWRIGHT_CHROME_CHANNEL', 'chrome'))
|
||||
parser.add_argument('--capture-types', default='xhr,fetch,document,other')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
args = parse_args()
|
||||
# normalize capture types to a set
|
||||
args.capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
|
||||
try:
|
||||
asyncio.run(run(args))
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
+106
-29
@@ -88,6 +88,8 @@ def main():
|
||||
parser.add_argument("--headless", action="store_true", help="Run headless")
|
||||
parser.add_argument("--capture-wait", type=int, default=8, help="Seconds to wait after reload to capture XHRs")
|
||||
parser.add_argument("--keep-open", action="store_true", help="Keep browser open after capture")
|
||||
parser.add_argument("--continuous", action="store_true", help="Continue capturing requests as you browse until interrupted")
|
||||
parser.add_argument("--no-prompt", action="store_true", help="Do not prompt before starting capture (useful for automated runs)")
|
||||
parser.add_argument("--username", help="STATdx username (or use STATDX_USERNAME env var)")
|
||||
parser.add_argument("--password", help="STATdx password (or use STATDX_PASSWORD env var)")
|
||||
parser.add_argument("--post-login-selector", default="#ds-app", help="Selector to wait for after login")
|
||||
@@ -101,23 +103,31 @@ def main():
|
||||
with sync_playwright() as p:
|
||||
# try to use installed Chrome first
|
||||
try:
|
||||
browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
|
||||
page = browser.pages[0] if browser.pages else browser.new_page()
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
|
||||
page = context.pages[0] if context.pages else context.new_page()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to launch Chrome channel: {}. Falling back to chromium bundle.", e)
|
||||
browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
page = browser.pages[0] if browser.pages else browser.new_page()
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
page = context.pages[0] if context.pages else context.new_page()
|
||||
|
||||
# attach CDP session to the page
|
||||
try:
|
||||
session = browser.new_cdp_session(page)
|
||||
session = context.new_cdp_session(page)
|
||||
except Exception as e:
|
||||
logger.error("Failed to create CDP session: {}", e)
|
||||
browser.close()
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
logger.info("Enabling Network domain on CDP")
|
||||
session.send("Network.enable")
|
||||
# Disable browser cache to force network requests (helps capture cached items)
|
||||
try:
|
||||
session.send("Network.setCacheDisabled", {"cacheDisabled": True})
|
||||
except Exception:
|
||||
logger.debug("Could not disable cache via CDP; continuing without cache disable")
|
||||
|
||||
seen = set()
|
||||
|
||||
@@ -137,7 +147,9 @@ def main():
|
||||
return
|
||||
seen.add(key)
|
||||
|
||||
# attempt to get body
|
||||
# attempt to get body; sometimes responseReceived doesn't allow
|
||||
# getResponseBody immediately, so we'll try but also rely on
|
||||
# a loadingFinished handler below.
|
||||
body = None
|
||||
base64_encoded = False
|
||||
try:
|
||||
@@ -146,7 +158,7 @@ def main():
|
||||
body = result.get("body")
|
||||
base64_encoded = bool(result.get("base64Encoded"))
|
||||
except Exception as e:
|
||||
logger.debug("Network.getResponseBody failed for {}: {}", url, e)
|
||||
logger.debug("Network.getResponseBody failed (responseReceived) for {}: {}", url, e)
|
||||
|
||||
# fallback: if no body, skip writing but still store metadata
|
||||
# build filenames
|
||||
@@ -196,7 +208,54 @@ def main():
|
||||
# subscribe to CDP event
|
||||
session.on("Network.responseReceived", on_response_received)
|
||||
|
||||
logger.info("Opening page: {}. Please log in if needed, then press ENTER to start capture.", args.url)
|
||||
# loadingFinished is a good fallback: some responses only have bodies
|
||||
# available after loadingFinished. We'll attempt to fetch body there
|
||||
# if not previously saved for the same requestId.
|
||||
def on_loading_finished(params):
|
||||
try:
|
||||
requestId = params.get("requestId")
|
||||
# call the same body retrieval logic via a small wrapper
|
||||
# create a fake params dict to reuse code
|
||||
fake = {"requestId": requestId, "response": {}}
|
||||
# attempt to get the body now
|
||||
try:
|
||||
result = session.send("Network.getResponseBody", {"requestId": requestId})
|
||||
except Exception:
|
||||
result = None
|
||||
if result:
|
||||
# build a minimal params for saving using response info from CDP
|
||||
resp = result
|
||||
# We don't have the URL here; try to fetch it via CDP - getRequestPostData isn't available.
|
||||
# Instead, reuse on_response_received's behavior by calling getResponseBody and writing
|
||||
# files directly here when possible.
|
||||
try:
|
||||
body = result.get("body")
|
||||
base64_encoded = bool(result.get("base64Encoded"))
|
||||
# fetch additional response info via Network.getResponseBody? we already have body.
|
||||
# write a minimal metadata file
|
||||
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
||||
safe = sanitize_fn(requestId)
|
||||
h = hashlib.sha1((requestId + str(time.time())).encode("utf-8")).hexdigest()[:8]
|
||||
ext = "bin"
|
||||
try:
|
||||
data = base64.b64decode(body) if base64_encoded else body.encode("utf-8")
|
||||
body_filename = f"{safe}_{h}_{ts}.{ext}"
|
||||
write_file(out_dir / body_filename, data)
|
||||
meta = {"requestId": requestId, "timestamp": ts, "response_body_file": str(out_dir / body_filename)}
|
||||
meta_filename = f"{safe}_{h}_{ts}.json"
|
||||
with open(out_dir / meta_filename, "w", encoding="utf-8") as mf:
|
||||
json.dump(meta, mf, ensure_ascii=False, indent=2)
|
||||
logger.info("Saved body via loadingFinished: {}", body_filename)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Error in loadingFinished handler")
|
||||
|
||||
session.on("Network.loadingFinished", on_loading_finished)
|
||||
|
||||
logger.info("Opening page: {}.", args.url)
|
||||
page.goto(args.url)
|
||||
|
||||
# Automatic login if credentials provided
|
||||
@@ -256,28 +315,46 @@ def main():
|
||||
except Exception:
|
||||
logger.exception('Automatic login attempt failed')
|
||||
|
||||
input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n")
|
||||
# Start capture: either prompt the user or begin immediately depending on flags
|
||||
try:
|
||||
if not args.no_prompt and not args.continuous:
|
||||
try:
|
||||
input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n")
|
||||
except Exception:
|
||||
# non-interactive; proceed
|
||||
pass
|
||||
|
||||
logger.info("Reloading page to trigger XHRs and network activity")
|
||||
page.reload()
|
||||
|
||||
# wait a bit to let XHRs fire and be processed by CDP
|
||||
wait = args.capture_wait
|
||||
logger.info("Waiting {} seconds to collect responses...", wait)
|
||||
time.sleep(wait)
|
||||
|
||||
logger.info("Capture pass complete. Metadata and bodies written to {}", out_dir)
|
||||
|
||||
if args.keep_open:
|
||||
logger.info("Keeping browser open. Close manually when done.")
|
||||
logger.info("Reloading page to trigger XHRs and network activity")
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Received interrupt, closing.")
|
||||
browser.close()
|
||||
else:
|
||||
browser.close()
|
||||
page.reload()
|
||||
except Exception:
|
||||
logger.exception('Failed to reload page; continuing capture')
|
||||
|
||||
# wait a bit to let XHRs fire and be processed by CDP
|
||||
wait = args.capture_wait
|
||||
logger.info("Waiting {} seconds to collect initial responses...", wait)
|
||||
time.sleep(wait)
|
||||
|
||||
logger.info("Initial capture pass complete. Metadata and bodies written to {}", out_dir)
|
||||
|
||||
if args.continuous:
|
||||
logger.info("Continuous capture enabled: saving responses as you browse. Press Ctrl+C to stop.")
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Interrupted by user; closing browser and exiting")
|
||||
else:
|
||||
logger.info("One-shot capture complete.")
|
||||
|
||||
finally:
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Save full page HTML snapshots after page loads and periodically while you browse.
|
||||
|
||||
This script launches a persistent Playwright profile (so you can log in),
|
||||
optionally attempts automatic login, then either performs a one-shot snapshot
|
||||
or continues saving snapshots as you browse. Snapshots are de-duplicated by
|
||||
content hash to avoid writing identical files repeatedly.
|
||||
|
||||
Usage examples:
|
||||
# manual login, one-shot snapshot after reload
|
||||
python scrapers/save_page_snapshots.py --url https://app.statdx.com/ --output-dir xhr_captured
|
||||
|
||||
# auto-login and continuous snapshots while you browse
|
||||
python scrapers/save_page_snapshots.py --continuous --username you --password secret --output-dir xhr_captured
|
||||
|
||||
"""
|
||||
from pathlib import Path
|
||||
import argparse
|
||||
import time
|
||||
import hashlib
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
from loguru import logger
|
||||
except Exception:
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("save_page_snapshots")
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# load .env
|
||||
load_dotenv()
|
||||
|
||||
|
||||
def sanitize_filename(s: str) -> str:
|
||||
safe = []
|
||||
for ch in s:
|
||||
if ch.isalnum() or ch in ('-', '_', '.'):
|
||||
safe.append(ch)
|
||||
else:
|
||||
safe.append('-')
|
||||
name = ''.join(safe)
|
||||
if len(name) > 180:
|
||||
h = hashlib.sha1(s.encode('utf-8')).hexdigest()[:8]
|
||||
name = name[:120] + '-' + h
|
||||
return name
|
||||
|
||||
|
||||
def write_text(path: Path, text: str):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
|
||||
|
||||
def write_json(path: Path, obj):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, 'w', encoding='utf-8') as f:
|
||||
json.dump(obj, f, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def now_ts():
|
||||
return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
|
||||
def snapshot_page(page, out_dir: Path, index_path: Path, last_hashes: dict, min_interval: float = 3.0):
|
||||
"""Capture page content and write HTML + meta if content changed or enough time passed."""
|
||||
try:
|
||||
url = page.url
|
||||
except Exception:
|
||||
url = 'unknown'
|
||||
|
||||
try:
|
||||
content = page.content()
|
||||
except Exception as e:
|
||||
logger.exception('Failed to get page.content(): {}', e)
|
||||
return None
|
||||
|
||||
h = hashlib.sha1(content.encode('utf-8')).hexdigest()
|
||||
prev = last_hashes.get(url)
|
||||
ts = now_ts()
|
||||
|
||||
# If unchanged and we saved recently, skip
|
||||
if prev and prev.get('hash') == h and (time.time() - prev.get('ts', 0)) < min_interval:
|
||||
logger.debug('Snapshot skipped (no change) for {}', url)
|
||||
return None
|
||||
|
||||
# prepare filenames
|
||||
name = sanitize_filename(url.replace('https://', '').replace('http://', ''))
|
||||
short = h[:8]
|
||||
html_name = f"{name}_{short}_{ts}.html"
|
||||
meta_name = f"{name}_{short}_{ts}.json"
|
||||
|
||||
html_path = out_dir / html_name
|
||||
meta_path = out_dir / meta_name
|
||||
|
||||
try:
|
||||
write_text(html_path, content)
|
||||
except Exception as e:
|
||||
logger.exception('Failed to write HTML snapshot: {}', e)
|
||||
return None
|
||||
|
||||
meta = {
|
||||
'url': url,
|
||||
'timestamp': ts,
|
||||
'file': str(html_path),
|
||||
'sha1': h,
|
||||
'title': (page.title() if page else None),
|
||||
}
|
||||
try:
|
||||
write_json(meta_path, meta)
|
||||
except Exception:
|
||||
logger.exception('Failed to write meta file')
|
||||
|
||||
# append to index
|
||||
try:
|
||||
entry = {'url': url, 'timestamp': ts, 'html_file': str(html_path), 'meta_file': str(meta_path), 'sha1': h}
|
||||
with open(index_path, 'a', encoding='utf-8') as idx:
|
||||
idx.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||||
except Exception:
|
||||
logger.exception('Failed to append to index')
|
||||
|
||||
last_hashes[url] = {'hash': h, 'ts': time.time()}
|
||||
logger.info('Saved snapshot: {} ({} bytes)', html_path.name, len(content))
|
||||
return html_path
|
||||
|
||||
|
||||
def attempt_autologin(page, username: str, password: str, post_login_selector: str, wait_after: float = 3.0):
|
||||
if not username or not password:
|
||||
return False
|
||||
logger.info('Attempting automatic login')
|
||||
# Common selectors for STATdx and similar pages
|
||||
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
|
||||
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
||||
|
||||
user_sel = None
|
||||
for sel in user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
pass_sel = None
|
||||
for sel in pass_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
pass_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel or not pass_sel:
|
||||
logger.warning('Could not find username/password fields for auto-login')
|
||||
return False
|
||||
|
||||
try:
|
||||
page.fill(user_sel, username)
|
||||
time.sleep(0.1)
|
||||
page.fill(pass_sel, password)
|
||||
time.sleep(0.1)
|
||||
# try submit
|
||||
submit_selectors = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
|
||||
clicked = False
|
||||
for s in submit_selectors:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
logger.info('Clicked submit button using selector: {}', s)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if not clicked:
|
||||
page.press(pass_sel, 'Enter')
|
||||
|
||||
# wait for post-login selector
|
||||
try:
|
||||
if post_login_selector:
|
||||
page.wait_for_selector(post_login_selector, timeout=10000)
|
||||
except Exception:
|
||||
logger.info('Post-login selector not found within timeout')
|
||||
time.sleep(wait_after)
|
||||
return True
|
||||
except Exception:
|
||||
logger.exception('Auto-login failed')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--url', '-u', default='https://app.statdx.com/')
|
||||
parser.add_argument('--profile', '-p', default='playwright_profile')
|
||||
parser.add_argument('--output-dir', '-o', default='xhr_captured')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
parser.add_argument('--continuous', action='store_true', help='Keep capturing while you browse until interrupted')
|
||||
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait before initial snapshot')
|
||||
parser.add_argument('--poll-interval', type=float, default=5.0, help='Periodic snapshot interval when continuous')
|
||||
parser.add_argument('--min-interval', type=float, default=3.0, help='Minimum seconds between snapshots for same URL')
|
||||
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
|
||||
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
|
||||
parser.add_argument('--post-login-selector', default='#ds-app')
|
||||
parser.add_argument('--no-prompt', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
index_path = out_dir / 'snapshot_index.jsonl'
|
||||
|
||||
username = args.username or os.getenv('STATDX_USERNAME')
|
||||
password = args.password or os.getenv('STATDX_PASSWORD')
|
||||
|
||||
last_hashes = {}
|
||||
|
||||
with sync_playwright() as p:
|
||||
try:
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome')
|
||||
except Exception:
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
page = context.new_page()
|
||||
logger.info('Opening %s', args.url)
|
||||
page.goto(args.url)
|
||||
|
||||
# attempt auto-login
|
||||
if username and password:
|
||||
attempt_autologin(page, username, password, args.post_login_selector)
|
||||
|
||||
# wait before initial snapshot to let in-flight XHRs populate DOM
|
||||
logger.info('Waiting %.1f sec before initial snapshot', args.capture_wait)
|
||||
time.sleep(args.capture_wait)
|
||||
|
||||
# initial snapshot
|
||||
snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
|
||||
|
||||
if args.continuous:
|
||||
logger.info('Continuous snapshot mode: saving on navigation and every %.1f sec', args.poll_interval)
|
||||
|
||||
# on navigation events, take a snapshot (debounced by min_interval)
|
||||
def on_load():
|
||||
try:
|
||||
snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
|
||||
except Exception:
|
||||
logger.exception('Error during on_load snapshot')
|
||||
|
||||
page.on('load', lambda: on_load())
|
||||
page.on('domcontentloaded', lambda: on_load())
|
||||
page.on('framenavigated', lambda frame: on_load())
|
||||
|
||||
try:
|
||||
# periodic polling snapshot
|
||||
while True:
|
||||
time.sleep(args.poll_interval)
|
||||
snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
|
||||
except KeyboardInterrupt:
|
||||
logger.info('Interrupted by user, closing')
|
||||
else:
|
||||
# one-shot: save a final snapshot after a short delay to capture dynamic updates
|
||||
try:
|
||||
time.sleep(1.0)
|
||||
snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval)
|
||||
except Exception:
|
||||
logger.exception('Error taking final snapshot')
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -22,7 +22,6 @@ Notes / limitations:
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import threading
|
||||
import sys
|
||||
@@ -32,9 +31,12 @@ from urllib.parse import urlparse
|
||||
from dotenv import load_dotenv
|
||||
import mimetypes
|
||||
|
||||
from loguru import logger
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
logger.remove()
|
||||
logger.add(sys.stderr, level="INFO")
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
@@ -79,7 +81,7 @@ def auto_scroll(page, step_ms: int = 50, step_px: int = 300):
|
||||
step_ms,
|
||||
)
|
||||
except Exception:
|
||||
logging.exception('auto_scroll failed')
|
||||
logger.exception('auto_scroll failed')
|
||||
|
||||
|
||||
def save_response(resp, out_dir: str):
|
||||
@@ -112,7 +114,7 @@ def save_response(resp, out_dir: str):
|
||||
filename_base = sanitize_filename(url)
|
||||
out_path = os.path.join(out_dir, filename_base + ext)
|
||||
|
||||
logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0])
|
||||
logger.info(f'Saving {url} -> {out_path} (content-type: {ctype.split(";")[0]})')
|
||||
|
||||
if save_as_text:
|
||||
try:
|
||||
@@ -136,7 +138,7 @@ def save_response(resp, out_dir: str):
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.exception('Failed to save response: %s', e)
|
||||
logger.exception(f'Failed to save response: {e}')
|
||||
return False
|
||||
|
||||
|
||||
@@ -147,21 +149,21 @@ def save_page_snapshot(page, out_dir: str, reason: str = ''):
|
||||
filename_base = sanitize_filename(url)
|
||||
safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40]
|
||||
out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html")
|
||||
logging.info('Saving page snapshot: %s -> %s', url, out_path)
|
||||
logger.info(f'Saving page snapshot: {url} -> {out_path}')
|
||||
content = page.content()
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(content)
|
||||
# Screenshots are disabled (not saving visual snapshots per user request)
|
||||
return True
|
||||
except Exception:
|
||||
logging.exception('Failed to save page snapshot')
|
||||
logger.exception('Failed to save page snapshot')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright')
|
||||
parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses')
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory')
|
||||
parser.add_argument('--profile', '-p', default='playwright_profile', help='Persistent profile directory')
|
||||
parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain')
|
||||
parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)')
|
||||
parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot')
|
||||
@@ -182,7 +184,7 @@ def main():
|
||||
with sync_playwright() as p:
|
||||
# Use persistent context so cookies/localStorage persist between runs
|
||||
browser_type = p.chromium
|
||||
logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel)
|
||||
logger.info(f'Launching Chromium (headless={args.headless}) with profile: {args.profile}, channel={args.channel}')
|
||||
if args.channel:
|
||||
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
|
||||
else:
|
||||
@@ -198,6 +200,7 @@ def main():
|
||||
pass
|
||||
|
||||
def handle_response(resp):
|
||||
logger.debug(f'RESPONSE: {resp.url} ({resp.request.resource_type})')
|
||||
try:
|
||||
if args.domain not in resp.url:
|
||||
return
|
||||
@@ -214,12 +217,12 @@ def main():
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('auto-scroll failed')
|
||||
logger.exception('auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason=rtype)
|
||||
except Exception:
|
||||
logging.exception('snapshot after response failed')
|
||||
logger.exception('snapshot after response failed')
|
||||
except Exception:
|
||||
logging.exception('Error in response handler')
|
||||
logger.exception('Error in response handler')
|
||||
|
||||
|
||||
page.on('response', handle_response)
|
||||
@@ -227,19 +230,19 @@ def main():
|
||||
# Log console messages and page errors to help diagnose missing content
|
||||
def on_console(msg):
|
||||
try:
|
||||
logging.info('PAGE LOG [%s] %s', msg.type, msg.text)
|
||||
logger.info(f'PAGE LOG [{msg.type}] {msg.text}')
|
||||
except Exception:
|
||||
logging.info('PAGE LOG: %s', msg)
|
||||
logger.info(f'PAGE LOG: {msg}')
|
||||
|
||||
def on_page_error(exc):
|
||||
logging.error('PAGE ERROR: %s', exc)
|
||||
logger.error(f'PAGE ERROR: {exc}')
|
||||
|
||||
page.on('console', on_console)
|
||||
page.on('pageerror', on_page_error)
|
||||
|
||||
logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output)
|
||||
logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
|
||||
logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
|
||||
logger.info(f'Open the browser window and interact with the site. Captured responses will be saved to: {args.output}')
|
||||
logger.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
|
||||
logger.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
|
||||
|
||||
# Navigate to the domain root so you can log in manually if needed
|
||||
start_url = f'https://{args.domain}/'
|
||||
@@ -247,27 +250,27 @@ def main():
|
||||
# Optionally wait for a selector that indicates the page finished rendering
|
||||
if args.wait_for_selector:
|
||||
try:
|
||||
logging.info('Waiting for selector: %s', args.wait_for_selector)
|
||||
logger.info(f'Waiting for selector: {args.wait_for_selector}')
|
||||
page.wait_for_selector(args.wait_for_selector, timeout=20000)
|
||||
except Exception:
|
||||
logging.exception('Waiting for selector timed out')
|
||||
logger.exception('Waiting for selector timed out')
|
||||
|
||||
def stdin_watcher():
|
||||
# Run in a background thread so the main thread can keep the browser running
|
||||
logging.debug('stdin watcher started')
|
||||
logger.debug('stdin watcher started')
|
||||
for line in sys.stdin:
|
||||
cmd = line.strip().lower()
|
||||
if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save':
|
||||
logging.info('Manual snapshot requested via stdin')
|
||||
logger.info('Manual snapshot requested via stdin')
|
||||
try:
|
||||
if args.auto_scroll:
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('manual auto-scroll failed')
|
||||
logger.exception('manual auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason='manual')
|
||||
except Exception:
|
||||
logging.exception('manual snapshot failed')
|
||||
logger.exception('manual snapshot failed')
|
||||
|
||||
watcher = threading.Thread(target=stdin_watcher, daemon=True)
|
||||
watcher.start()
|
||||
@@ -277,7 +280,7 @@ def main():
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Shutting down...')
|
||||
logger.info('Shutting down...')
|
||||
finally:
|
||||
context.close()
|
||||
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
{
|
||||
"url": "https://app.statdx.com/csrf/token",
|
||||
"method": "GET",
|
||||
"resource_type": "xhr",
|
||||
"status": 200,
|
||||
"request_headers": {
|
||||
"sec-ch-ua-platform": "\"Linux\"",
|
||||
"referer": "https://app.statdx.com/",
|
||||
"x-requested-with": "XMLHttpRequest",
|
||||
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
"accept": "*/*",
|
||||
"sec-ch-ua": "\"Not=A?Brand\";v=\"24\", \"Chromium\";v=\"140\"",
|
||||
"sec-ch-ua-mobile": "?0"
|
||||
},
|
||||
"response_headers": {
|
||||
"version-pss-dsjs-config": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
||||
"version-pss-dsjs": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
||||
"content-encoding": "gzip",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"etag": "W/\"10c-vLpriK5k7wHbwVGauL6RBVcuBHQ\"",
|
||||
"x-ua-compatible": "IE=edge",
|
||||
"date": "Tue, 14 Oct 2025 12:41:21 GMT",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"vary": "Accept-Encoding",
|
||||
"version-pss-sparks-event-client": "2.1.1",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains",
|
||||
"content-security-policy": "script-src 'self' 'unsafe-eval' 'unsafe-inline' js-agent.newrelic.com static.cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; style-src 'self' 'unsafe-inline' fonts.googleapis.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; img-src 'self' data: app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com *.cloudfront.net; media-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com *.cloudfront.net; connect-src 'self' ws: cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; frame-src 'self' app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com; worker-src 'self' blob: scopus.com service.elsevier.com pendo.reaxys.com; object-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com",
|
||||
"cache-control": "no-cache, no-store",
|
||||
"tdm-reservation": "1",
|
||||
"x-ratelimit-reset": "1760445742",
|
||||
"hostname": "sdxapp3",
|
||||
"cf-ray": "98e726878fefbea1-LHR",
|
||||
"x-ratelimit-remaining": "499",
|
||||
"version-pss-cmejs": "9.3.3",
|
||||
"tdm-policy": "https://www.elsevier.com/tdm/tdmrep-policy.json",
|
||||
"x-ratelimit-limit": "500",
|
||||
"version-pss-authenticationjs": "7.1.0",
|
||||
"x-powered-by": "Express",
|
||||
"server": "cloudflare"
|
||||
},
|
||||
"timestamp": "20251014T124122Z",
|
||||
"request_post_data": null,
|
||||
"response_text": "{\"token\":\"c67a4a30b300096e11483fc08f9dcb69e679ef1152816abe6bcad85c694959d5ba2e29cab29da9e13a743b29daac792a44a0af3bcf7f0b6476c1691d52b01918472529e208a3431dc22a435435c7947b1212f546ae3b618ad1701e1f37f616c6f6a76cfa179fb82706566692759ca11b617ba0d9501d9ee88a7e1e6ac2c8be39\"}",
|
||||
"response_body_file": null
|
||||
}
|
||||
Reference in New Issue
Block a user