Ross
e3d081cbff
- Implement capture_with_cdp.py to capture all network response bodies using Chrome DevTools Protocol. - Create extract_sections.py to extract sections from STATdx snapshot HTML files into JSON Lines. - Add unpack_document_content.py to extract `documentHtml` from captured JSON bodies and save as HTML files. - Update .gitignore to include playwright_profile.
424 lines
20 KiB
Python
424 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""Launch the site in an actual installed Chrome via Playwright's 'chrome' channel.
|
|
|
|
This can help if the headful Playwright Chromium environment differs from your regular Chrome and the site
|
|
only fully renders in a real browser environment.
|
|
|
|
Usage:
|
|
python scrapers/statdx_play_chrome.py --profile .chrome_profile --url https://app.statdx.com/ --screenshot out.png
|
|
|
|
Notes:
|
|
- Requires Playwright and that the 'chrome' browser is available on your machine.
|
|
- If Playwright cannot find the channel, install Chrome or use the system browser path.
|
|
"""
|
|
|
|
import argparse
|
|
from loguru import logger
|
|
import os
|
|
from time import sleep
|
|
from urllib.parse import urlparse
|
|
import hashlib
|
|
import mimetypes
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from playwright.sync_api import sync_playwright
|
|
from dotenv import load_dotenv
|
|
|
|
# Load environment variables from .env in the repository root (if present)
|
|
load_dotenv()
|
|
|
|
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--profile', '-p', default='.chrome_profile')
|
|
parser.add_argument('--url', '-u', default='https://app.statdx.com/')
|
|
parser.add_argument('--domain', '-d', default='', help='Domain to filter captures')
|
|
parser.add_argument('--screenshot', '-s', default='chrome_page.png')
|
|
parser.add_argument('--wait-for', help='Optional selector to wait for before screenshot')
|
|
parser.add_argument('--headless', action='store_true')
|
|
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
|
|
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
|
|
parser.add_argument('--post-login-selector', default='#ds-app', help='Selector indicating successful login (default: #ds-app)')
|
|
parser.add_argument('--wait-after-login', type=float, default=3.0, help='Seconds to wait after login before snapshot')
|
|
parser.add_argument('--keep-open', action='store_true', help='Leave the browser open after actions')
|
|
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
|
|
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
|
|
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
|
|
parser.add_argument('--capture-types', default='xhr,fetch,document,image', help='Comma-separated list of resource types to capture (e.g. xhr,fetch,document,image)')
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(args.profile, exist_ok=True)
|
|
|
|
with sync_playwright() as p:
|
|
try:
|
|
logger.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
|
|
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
|
|
except Exception as e:
|
|
logger.error(f'Failed to launch Chrome channel: {e}')
|
|
logger.info('Falling back to default Chromium persistent context')
|
|
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
|
|
|
page = context.new_page()
|
|
page.goto(args.url)
|
|
|
|
# Automatic login if credentials provided (run before any capture)
|
|
username = args.username or os.getenv('STATDX_USERNAME')
|
|
password = args.password or os.getenv('STATDX_PASSWORD')
|
|
if username and password:
|
|
logger.info('Attempting automatic login using provided credentials')
|
|
# candidate selectors for username and password fields
|
|
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
|
|
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
|
|
|
user_sel = None
|
|
for sel in user_selectors:
|
|
try:
|
|
if page.query_selector(sel):
|
|
user_sel = sel
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
pass_sel = None
|
|
for sel in pass_selectors:
|
|
try:
|
|
if page.query_selector(sel):
|
|
pass_sel = sel
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not user_sel or not pass_sel:
|
|
logger.warning('Could not auto-detect username/password fields. Skipping automatic login.')
|
|
else:
|
|
try:
|
|
page.fill(user_sel, username)
|
|
sleep(0.1)
|
|
page.fill(pass_sel, password)
|
|
sleep(0.1)
|
|
# try submit buttons
|
|
submit_selectors = ['button.primary.submitSelector', 'button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
|
|
clicked = False
|
|
for s in submit_selectors:
|
|
try:
|
|
btn = page.query_selector(s)
|
|
if btn:
|
|
btn.click()
|
|
clicked = True
|
|
logger.info(f'Clicked submit button using selector: {s}')
|
|
break
|
|
except Exception:
|
|
continue
|
|
if not clicked:
|
|
logger.info('No submit button clicked; pressing Enter in password field')
|
|
page.press(pass_sel, 'Enter')
|
|
|
|
# wait for either navigation or a post-login selector
|
|
try:
|
|
if args.post_login_selector:
|
|
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
|
|
except Exception:
|
|
logger.info('Post-login selector not found within timeout')
|
|
|
|
# small extra wait to let JS render
|
|
sleep(args.wait_after_login)
|
|
except Exception:
|
|
logger.exception('Automatic login attempt failed')
|
|
|
|
# Setup capture if requested
|
|
captured = []
|
|
if args.capture:
|
|
os.makedirs(args.capture_output, exist_ok=True)
|
|
|
|
def sanitize_filename(url: str) -> str:
|
|
parsed = urlparse(url)
|
|
base = parsed.netloc + parsed.path
|
|
if parsed.query:
|
|
base += '?' + parsed.query
|
|
base = base.strip('/')
|
|
if not base:
|
|
base = parsed.netloc
|
|
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
|
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
|
return f"{safe[:200]}_{h}"
|
|
|
|
def save_binary(body: bytes, out_dir: str, url: str, ts: str, ctype: str):
|
|
ext = ''
|
|
try:
|
|
ext = mimetypes.guess_extension(ctype.split(';')[0]) or ''
|
|
except Exception:
|
|
ext = ''
|
|
fname = f"{sanitize_filename(url)}_{ts}{ext}"
|
|
path = os.path.join(out_dir, fname)
|
|
with open(path, 'wb') as fh:
|
|
fh.write(body)
|
|
return path
|
|
|
|
capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
|
|
|
|
def on_response(resp):
|
|
# lightweight debug log
|
|
logger.debug(f'Response: {resp.status} {resp.url}')
|
|
try:
|
|
req = resp.request
|
|
rtype = (req.resource_type or '').lower()
|
|
|
|
# Save images regardless of domain filter when image capture is enabled.
|
|
if rtype == 'image' and 'image' in capture_types:
|
|
allow_save = True
|
|
else:
|
|
# For non-image responses, enforce domain filter
|
|
if args.domain and args.domain not in req.url:
|
|
return
|
|
allow_save = rtype in capture_types
|
|
|
|
if not allow_save:
|
|
return
|
|
|
|
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
|
|
info = {
|
|
'url': req.url,
|
|
'method': req.method,
|
|
'resource_type': rtype,
|
|
'status': resp.status,
|
|
'request_headers': dict(req.headers),
|
|
'response_headers': dict(resp.headers),
|
|
'timestamp': ts,
|
|
}
|
|
|
|
# Request post data
|
|
try:
|
|
post = req.post_data
|
|
info['request_post_data'] = post
|
|
except Exception:
|
|
try:
|
|
info['request_post_data'] = req.post_data_text()
|
|
except Exception:
|
|
info['request_post_data'] = None
|
|
|
|
# Response body (always save to a standalone file)
|
|
ctype = (resp.headers.get('content-type') or '').lower()
|
|
body_path = None
|
|
excerpt = None
|
|
try:
|
|
# Document / HTML
|
|
if 'text/html' in ctype or rtype == 'document':
|
|
txt = resp.text()
|
|
ext = '.html'
|
|
body_name = f"{sanitize_filename(req.url)}_{ts}{ext}"
|
|
body_path = os.path.join(args.capture_output, body_name)
|
|
with open(body_path, 'w', encoding='utf-8') as fh:
|
|
fh.write(txt)
|
|
excerpt = txt[:400]
|
|
# JSON
|
|
elif 'application/json' in ctype or req.url.endswith('.json'):
|
|
txt = resp.text()
|
|
ext = '.json'
|
|
body_name = f"{sanitize_filename(req.url)}_{ts}{ext}"
|
|
body_path = os.path.join(args.capture_output, body_name)
|
|
# pretty-print JSON when possible
|
|
try:
|
|
parsed = json.loads(txt)
|
|
with open(body_path, 'w', encoding='utf-8') as fh:
|
|
json.dump(parsed, fh, indent=2)
|
|
except Exception:
|
|
with open(body_path, 'w', encoding='utf-8') as fh:
|
|
fh.write(txt)
|
|
excerpt = txt[:400]
|
|
# text/javascript or other text
|
|
elif ctype.startswith('text/') or 'javascript' in ctype:
|
|
txt = resp.text()
|
|
ext = '.txt'
|
|
body_name = f"{sanitize_filename(req.url)}_{ts}{ext}"
|
|
body_path = os.path.join(args.capture_output, body_name)
|
|
with open(body_path, 'w', encoding='utf-8') as fh:
|
|
fh.write(txt)
|
|
excerpt = txt[:400]
|
|
else:
|
|
# binary (images, fonts, etc.)
|
|
body = resp.body()
|
|
body_path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
|
excerpt = None
|
|
except Exception:
|
|
try:
|
|
# fallback to raw binary save
|
|
body = resp.body()
|
|
body_path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
|
except Exception:
|
|
logger.exception(f'Failed to read response body for {req.url}')
|
|
|
|
info['response_body_file'] = body_path
|
|
info['response_excerpt'] = excerpt
|
|
|
|
fname = f"{sanitize_filename(req.url)}_{ts}.json"
|
|
out_path = os.path.join(args.capture_output, fname)
|
|
with open(out_path, 'w', encoding='utf-8') as fh:
|
|
json.dump(info, fh, indent=2)
|
|
|
|
logger.info(f'Captured {rtype}: {req.url} -> {out_path} (body: {body_path})')
|
|
captured.append(out_path)
|
|
# append to a simple JSONL index for quick searching
|
|
try:
|
|
index_entry = {
|
|
'url': req.url,
|
|
'resource_type': rtype,
|
|
'timestamp': ts,
|
|
'body_file': body_path,
|
|
'meta_file': out_path,
|
|
'excerpt': excerpt,
|
|
}
|
|
index_path = os.path.join(args.capture_output, 'capture_index.jsonl')
|
|
with open(index_path, 'a', encoding='utf-8') as idx:
|
|
idx.write(json.dumps(index_entry, ensure_ascii=False) + '\n')
|
|
except Exception:
|
|
logger.exception('Failed to write capture index entry')
|
|
except Exception:
|
|
logger.exception('Error in capture on_response')
|
|
|
|
page.on('response', on_response)
|
|
|
|
# Wait for user to start capture so they can log in / navigate manually if needed.
|
|
try:
|
|
input('Press Enter to start capture (the page will be reloaded to trigger XHR/fetch requests)...')
|
|
except Exception:
|
|
# non-interactive environments may raise; continue
|
|
pass
|
|
|
|
logger.info('Reloading page to trigger API calls...')
|
|
try:
|
|
page.reload()
|
|
except Exception:
|
|
logger.exception('Failed to reload page; continuing capture')
|
|
|
|
# Try to auto-scroll to trigger lazy loading
|
|
try:
|
|
page.evaluate(
|
|
"""
|
|
async () => {
|
|
const wait = ms => new Promise(r => setTimeout(r, ms));
|
|
const doc = document.scrollingElement || document.documentElement;
|
|
let total = 0;
|
|
while (total < doc.scrollHeight) {
|
|
doc.scrollBy(0, 400);
|
|
total += 400;
|
|
await wait(150);
|
|
}
|
|
}
|
|
"""
|
|
)
|
|
except Exception:
|
|
logger.exception('auto-scroll failed during capture')
|
|
|
|
# Wait for the configured capture window so responses arrive and are saved
|
|
import time as _time
|
|
logger.info('Waiting %.1f seconds for responses...', args.capture_wait)
|
|
_time.sleep(args.capture_wait)
|
|
|
|
# Save a final rendered page snapshot (HTML)
|
|
try:
|
|
html = page.content()
|
|
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
|
fname = sanitize_filename(page.url)
|
|
snap_path = os.path.join(args.capture_output, f'snapshot_after_capture_{fname}_{ts}.html')
|
|
with open(snap_path, 'w', encoding='utf-8') as fh:
|
|
fh.write(html)
|
|
logger.info('Saved final page snapshot: %s', snap_path)
|
|
except Exception:
|
|
logger.exception('Failed to save final snapshot after capture')
|
|
|
|
# Automatic login if credentials provided
|
|
username = args.username or os.getenv('STATDX_USERNAME')
|
|
password = args.password or os.getenv('STATDX_PASSWORD')
|
|
if username and password:
|
|
logger.info('Attempting automatic login using provided credentials')
|
|
# candidate selectors for username and password fields
|
|
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
|
|
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
|
|
|
user_sel = None
|
|
for sel in user_selectors:
|
|
try:
|
|
if page.query_selector(sel):
|
|
user_sel = sel
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
pass_sel = None
|
|
for sel in pass_selectors:
|
|
try:
|
|
if page.query_selector(sel):
|
|
pass_sel = sel
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not user_sel or not pass_sel:
|
|
logger.warning('Could not auto-detect username/password fields. Skipping automatic login.')
|
|
else:
|
|
try:
|
|
page.fill(user_sel, username)
|
|
sleep(0.1)
|
|
page.fill(pass_sel, password)
|
|
sleep(0.1)
|
|
# try submit buttons
|
|
submit_selectors = ['button.primary.submitSelector', 'button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
|
|
clicked = False
|
|
for s in submit_selectors:
|
|
try:
|
|
btn = page.query_selector(s)
|
|
if btn:
|
|
btn.click()
|
|
clicked = True
|
|
logger.info(f'Clicked submit button using selector: {s}')
|
|
break
|
|
except Exception:
|
|
continue
|
|
if not clicked:
|
|
logger.info('No submit button clicked; pressing Enter in password field')
|
|
page.press(pass_sel, 'Enter')
|
|
|
|
# wait for either navigation or a post-login selector
|
|
try:
|
|
if args.post_login_selector:
|
|
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
|
|
except Exception:
|
|
logger.info('Post-login selector not found within timeout')
|
|
|
|
# small extra wait to let JS render
|
|
sleep(args.wait_after_login)
|
|
except Exception:
|
|
logger.exception('Automatic login attempt failed')
|
|
|
|
if args.wait_for:
|
|
try:
|
|
logger.info(f'Waiting for selector: {args.wait_for}')
|
|
page.wait_for_selector(args.wait_for, timeout=20000)
|
|
except Exception:
|
|
logger.exception('wait_for selector timed out')
|
|
|
|
# give some time for JS to finish
|
|
page.wait_for_timeout(1500)
|
|
|
|
# Screenshots are disabled per user request
|
|
|
|
if args.keep_open:
|
|
logger.info('Leaving browser open for manual inspection. Close the browser window to finish.')
|
|
try:
|
|
# keep running until the user presses Enter in the terminal
|
|
input('Browser is open. Press Enter here to close and exit...')
|
|
except Exception:
|
|
pass
|
|
|
|
try:
|
|
context.close()
|
|
except Exception:
|
|
logger.exception('Error closing context')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|