Ross
491fca2f42
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
220 lines
8.5 KiB
Python
220 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Capture-after-login helper
|
|
|
|
Usage:
|
|
1. Run this script. It will open a visible browser with a persistent profile (so you can log in manually).
|
|
2. Log in to STATdx in the opened browser and navigate to the topics page normally.
|
|
3. Switch back to the terminal and press Enter. The script will reload the topics URL to trigger API calls,
|
|
capture XHR/fetch response bodies, save them under the `output` directory, and dump cookies and localStorage.
|
|
|
|
Example:
|
|
python scrapers/statdx_capture_after_login.py --profile .playwright_profile --output captured --domain app.statdx.com --topics-url https://app.statdx.com/
|
|
|
|
This is intended to capture the API JSON that the client uses to render pages after authentication.
|
|
"""
|
|
|
|
import os
|
|
import argparse
|
|
import logging
|
|
import time
|
|
import json
|
|
import hashlib
|
|
from datetime import datetime, timezone
|
|
from urllib.parse import urlparse
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
|
|
|
|
|
def sanitize_filename(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
base = parsed.netloc + parsed.path
|
|
if parsed.query:
|
|
base += '?' + parsed.query
|
|
base = base.strip('/')
|
|
if not base:
|
|
base = parsed.netloc
|
|
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
|
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
|
return f"{safe[:200]}_{h}"
|
|
|
|
|
|
def dump_storage_and_cookies(context, page, out_dir: str):
|
|
try:
|
|
cookies = context.cookies()
|
|
with open(os.path.join(out_dir, 'cookies.json'), 'w', encoding='utf-8') as fh:
|
|
json.dump(cookies, fh, indent=2)
|
|
logging.info('Saved cookies to cookies.json')
|
|
except Exception:
|
|
logging.exception('Failed to dump cookies')
|
|
|
|
try:
|
|
local = page.evaluate('()=> JSON.stringify(localStorage)')
|
|
session = page.evaluate('()=> JSON.stringify(sessionStorage)')
|
|
with open(os.path.join(out_dir, 'localStorage.json'), 'w', encoding='utf-8') as fh:
|
|
fh.write(local)
|
|
with open(os.path.join(out_dir, 'sessionStorage.json'), 'w', encoding='utf-8') as fh:
|
|
fh.write(session)
|
|
logging.info('Saved localStorage and sessionStorage')
|
|
except Exception:
|
|
logging.exception('Failed to dump storage')
|
|
|
|
|
|
def save_response_body(resp, out_dir: str):
|
|
try:
|
|
url = resp.url
|
|
ctype = resp.headers.get('content-type', '')
|
|
fname = sanitize_filename(url)
|
|
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
if 'application/json' in ctype or url.endswith('.json'):
|
|
try:
|
|
text = resp.text()
|
|
except Exception:
|
|
text = resp.body().decode('utf-8', errors='replace')
|
|
path = os.path.join(out_dir, f"{fname}_{ts}.json")
|
|
with open(path, 'w', encoding='utf-8') as fh:
|
|
fh.write(text)
|
|
logging.info('Saved JSON response: %s', path)
|
|
return True
|
|
elif 'text/html' in ctype or resp.request.resource_type == 'document':
|
|
try:
|
|
text = resp.text()
|
|
except Exception:
|
|
text = resp.body().decode('utf-8', errors='replace')
|
|
path = os.path.join(out_dir, f"{fname}_{ts}.html")
|
|
with open(path, 'w', encoding='utf-8') as fh:
|
|
fh.write(text)
|
|
logging.info('Saved HTML response: %s', path)
|
|
return True
|
|
elif ctype.startswith('text/'):
|
|
try:
|
|
text = resp.text()
|
|
except Exception:
|
|
text = resp.body().decode('utf-8', errors='replace')
|
|
path = os.path.join(out_dir, f"{fname}_{ts}.txt")
|
|
with open(path, 'w', encoding='utf-8') as fh:
|
|
fh.write(text)
|
|
logging.info('Saved text response: %s', path)
|
|
return True
|
|
else:
|
|
# Save binary content (images, fonts, etc.)
|
|
try:
|
|
body = resp.body()
|
|
path = os.path.join(out_dir, f"{fname}_{ts}")
|
|
# try to add extension from content-type
|
|
ctype_main = resp.headers.get('content-type', '').split(';')[0]
|
|
try:
|
|
import mimetypes
|
|
|
|
ext = mimetypes.guess_extension(ctype_main) or ''
|
|
except Exception:
|
|
ext = ''
|
|
if ext:
|
|
path += ext
|
|
with open(path, 'wb') as fh:
|
|
fh.write(body)
|
|
logging.info('Saved binary response: %s', path)
|
|
return True
|
|
except Exception:
|
|
logging.exception('Failed to save binary response')
|
|
return False
|
|
except Exception:
|
|
logging.exception('Failed to save response body')
|
|
return False
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--profile', '-p', default='.playwright_profile')
|
|
parser.add_argument('--output', '-o', default='captured')
|
|
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
|
parser.add_argument('--topics-url', default='https://app.statdx.com/')
|
|
parser.add_argument('--wait', type=float, default=8.0, help='Seconds to wait after reload to capture responses')
|
|
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(args.output, exist_ok=True)
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium
|
|
logging.info('Launching persistent context with profile: %s (channel=%s)', args.profile, args.channel)
|
|
if args.channel:
|
|
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False, channel=args.channel)
|
|
else:
|
|
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False)
|
|
page = context.new_page()
|
|
|
|
captured = []
|
|
|
|
def on_response(resp):
|
|
try:
|
|
if args.domain not in resp.url:
|
|
return
|
|
rtype = resp.request.resource_type
|
|
# focus on XHR/fetch and document
|
|
if rtype in ('xhr', 'fetch', 'document'):
|
|
ok = save_response_body(resp, args.output)
|
|
if ok:
|
|
captured.append(resp.url)
|
|
except Exception:
|
|
logging.exception('response handler error')
|
|
|
|
page.on('response', on_response)
|
|
|
|
logging.info('Open browser and log in if needed. Navigate to the topics page. Then return to the terminal and press Enter to trigger capture.')
|
|
page.goto(args.topics_url)
|
|
|
|
input('Press Enter when you are logged in and on the page you want to capture...')
|
|
|
|
# Reload to trigger API calls while our handler is active
|
|
logging.info('Reloading topics URL to trigger API calls...')
|
|
page.reload()
|
|
|
|
# Optionally auto-scroll to trigger lazy loads
|
|
try:
|
|
page.evaluate(
|
|
"""
|
|
async () => {
|
|
const wait = ms => new Promise(r => setTimeout(r, ms));
|
|
const doc = document.scrollingElement || document.documentElement;
|
|
let total = 0;
|
|
while (total < doc.scrollHeight) {
|
|
doc.scrollBy(0, 400);
|
|
total += 400;
|
|
await wait(150);
|
|
}
|
|
}
|
|
"""
|
|
)
|
|
except Exception:
|
|
logging.exception('auto-scroll failed during capture')
|
|
|
|
logging.info('Waiting %.1f seconds for responses...', args.wait)
|
|
time.sleep(args.wait)
|
|
|
|
dump_storage_and_cookies(context, page, args.output)
|
|
|
|
# Save a final snapshot of the rendered page
|
|
try:
|
|
html = page.content()
|
|
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
fname = sanitize_filename(page.url)
|
|
snap_path = os.path.join(args.output, f'snapshot_after_login_{fname}_{ts}.html')
|
|
with open(snap_path, 'w', encoding='utf-8') as fh:
|
|
fh.write(html)
|
|
logging.info('Saved final page snapshot: %s', snap_path)
|
|
# screenshots disabled per user request
|
|
except Exception:
|
|
logging.exception('Failed to save final snapshot')
|
|
|
|
try:
|
|
context.close()
|
|
except Exception:
|
|
logging.exception('Error closing context')
|
|
|
|
logging.info('Done. Captured %d responses.', len(captured))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|