From 496d7a6523daa9b06f41a2584fd13a25a261db18 Mon Sep 17 00:00:00 2001 From: Ross Date: Tue, 14 Oct 2025 14:23:47 +0100 Subject: [PATCH] Update domain argument default value and enhance XHR capture to save HTML responses as files --- scrapers/statdx_play_chrome.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scrapers/statdx_play_chrome.py b/scrapers/statdx_play_chrome.py index e9e11e8..5370443 100644 --- a/scrapers/statdx_play_chrome.py +++ b/scrapers/statdx_play_chrome.py @@ -33,7 +33,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--profile', '-p', default='.chrome_profile') parser.add_argument('--url', '-u', default='https://app.statdx.com/') - parser.add_argument('--domain', '-d', default='app.statdx.com', help='Domain to filter captures') + parser.add_argument('--domain', '-d', default='', help='Domain to filter captures') parser.add_argument('--screenshot', '-s', default='chrome_page.png') parser.add_argument('--wait-for', help='Optional selector to wait for before screenshot') parser.add_argument('--headless', action='store_true') @@ -137,7 +137,17 @@ def main(): # Response body (text or binary) ctype = resp.headers.get('content-type', '') try: - if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype: + # Save HTML/document responses as separate .html files + if 'text/html' in ctype or rtype == 'document': + txt = resp.text() + html_name = f"{sanitize_filename(req.url)}_{ts}.html" + html_path = os.path.join(args.capture_output, html_name) + with open(html_path, 'w', encoding='utf-8') as fh: + fh.write(txt) + info['response_text'] = None + info['response_body_file'] = html_path + elif 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype: + # keep small textual responses inline in the metadata JSON txt = resp.text() info['response_text'] = txt info['response_body_file'] = None