ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Implement passive XHR/fetch capture using Playwright

Browse Source 
- Added `capture_passive_playwright.py` for synchronous passive capture of XHR/fetch responses.
- Added `capture_passive_playwright_async.py` for asynchronous passive capture using Playwright's async API.
- Introduced `save_page_snapshots.py` to save full page HTML snapshots with deduplication.
- Removed obsolete CSRF token capture JSON file.
This commit is contained in:
Ross
2025-10-14 19:48:01 +01:00
parent 683dd06643
commit adc21f5795
7 changed files with 1181 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/capture_with_cdp.py
+106 -29
View File
@@ -88,6 +88,8 @@ def main():
parser.add_argument("--headless", action="store_true", help="Run headless")
parser.add_argument("--capture-wait", type=int, default=8, help="Seconds to wait after reload to capture XHRs")
parser.add_argument("--keep-open", action="store_true", help="Keep browser open after capture")
parser.add_argument("--continuous", action="store_true", help="Continue capturing requests as you browse until interrupted")
parser.add_argument("--no-prompt", action="store_true", help="Do not prompt before starting capture (useful for automated runs)")
parser.add_argument("--username", help="STATdx username (or use STATDX_USERNAME env var)")
parser.add_argument("--password", help="STATdx password (or use STATDX_PASSWORD env var)")
parser.add_argument("--post-login-selector", default="#ds-app", help="Selector to wait for after login")
@@ -101,23 +103,31 @@ def main():
with sync_playwright() as p:
# try to use installed Chrome first
try:
browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
page = browser.pages[0] if browser.pages else browser.new_page()
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome")
page = context.pages[0] if context.pages else context.new_page()
except Exception as e:
logger.warning("Failed to launch Chrome channel: {}. Falling back to chromium bundle.", e)
browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
page = browser.pages[0] if browser.pages else browser.new_page()
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
page = context.pages[0] if context.pages else context.new_page()
# attach CDP session to the page
try:
session = browser.new_cdp_session(page)
session = context.new_cdp_session(page)
except Exception as e:
logger.error("Failed to create CDP session: {}", e)
browser.close()
try:
context.close()
except Exception:
pass
return
logger.info("Enabling Network domain on CDP")
session.send("Network.enable")
# Disable browser cache to force network requests (helps capture cached items)
try:
session.send("Network.setCacheDisabled", {"cacheDisabled": True})
except Exception:
logger.debug("Could not disable cache via CDP; continuing without cache disable")
seen = set()
@@ -137,7 +147,9 @@ def main():
return
seen.add(key)
# attempt to get body
# attempt to get body; sometimes responseReceived doesn't allow
# getResponseBody immediately, so we'll try but also rely on
# a loadingFinished handler below.
body = None
base64_encoded = False
try:
@@ -146,7 +158,7 @@ def main():
body = result.get("body")
base64_encoded = bool(result.get("base64Encoded"))
except Exception as e:
logger.debug("Network.getResponseBody failed for {}: {}", url, e)
logger.debug("Network.getResponseBody failed (responseReceived) for {}: {}", url, e)
# fallback: if no body, skip writing but still store metadata
# build filenames
@@ -196,7 +208,54 @@ def main():
# subscribe to CDP event
session.on("Network.responseReceived", on_response_received)
logger.info("Opening page: {}. Please log in if needed, then press ENTER to start capture.", args.url)
# loadingFinished is a good fallback: some responses only have bodies
# available after loadingFinished. We'll attempt to fetch body there
# if not previously saved for the same requestId.
def on_loading_finished(params):
try:
requestId = params.get("requestId")
# call the same body retrieval logic via a small wrapper
# create a fake params dict to reuse code
fake = {"requestId": requestId, "response": {}}
# attempt to get the body now
try:
result = session.send("Network.getResponseBody", {"requestId": requestId})
except Exception:
result = None
if result:
# build a minimal params for saving using response info from CDP
resp = result
# We don't have the URL here; try to fetch it via CDP - getRequestPostData isn't available.
# Instead, reuse on_response_received's behavior by calling getResponseBody and writing
# files directly here when possible.
try:
body = result.get("body")
base64_encoded = bool(result.get("base64Encoded"))
# fetch additional response info via Network.getResponseBody? we already have body.
# write a minimal metadata file
ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
safe = sanitize_fn(requestId)
h = hashlib.sha1((requestId + str(time.time())).encode("utf-8")).hexdigest()[:8]
ext = "bin"
try:
data = base64.b64decode(body) if base64_encoded else body.encode("utf-8")
body_filename = f"{safe}_{h}_{ts}.{ext}"
write_file(out_dir / body_filename, data)
meta = {"requestId": requestId, "timestamp": ts, "response_body_file": str(out_dir / body_filename)}
meta_filename = f"{safe}_{h}_{ts}.json"
with open(out_dir / meta_filename, "w", encoding="utf-8") as mf:
json.dump(meta, mf, ensure_ascii=False, indent=2)
logger.info("Saved body via loadingFinished: {}", body_filename)
except Exception:
pass
except Exception:
pass
except Exception:
logger.exception("Error in loadingFinished handler")
session.on("Network.loadingFinished", on_loading_finished)
logger.info("Opening page: {}.", args.url)
page.goto(args.url)
# Automatic login if credentials provided
@@ -256,28 +315,46 @@ def main():
except Exception:
logger.exception('Automatic login attempt failed')
input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n")
# Start capture: either prompt the user or begin immediately depending on flags
try:
if not args.no_prompt and not args.continuous:
try:
input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n")
except Exception:
# non-interactive; proceed
pass
logger.info("Reloading page to trigger XHRs and network activity")
page.reload()
# wait a bit to let XHRs fire and be processed by CDP
wait = args.capture_wait
logger.info("Waiting {} seconds to collect responses...", wait)
time.sleep(wait)
logger.info("Capture pass complete. Metadata and bodies written to {}", out_dir)
if args.keep_open:
logger.info("Keeping browser open. Close manually when done.")
logger.info("Reloading page to trigger XHRs and network activity")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Received interrupt, closing.")
browser.close()
else:
browser.close()
page.reload()
except Exception:
logger.exception('Failed to reload page; continuing capture')
# wait a bit to let XHRs fire and be processed by CDP
wait = args.capture_wait
logger.info("Waiting {} seconds to collect initial responses...", wait)
time.sleep(wait)
logger.info("Initial capture pass complete. Metadata and bodies written to {}", out_dir)
if args.continuous:
logger.info("Continuous capture enabled: saving responses as you browse. Press Ctrl+C to stop.")
try:
while True:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Interrupted by user; closing browser and exiting")
else:
logger.info("One-shot capture complete.")
finally:
try:
context.close()
except Exception:
try:
context.close()
except Exception:
pass
if __name__ == "__main__":