Update domain argument default value and enhance XHR capture to save HTML responses as files

scrapers/statdx_play_chrome.py

@@ -33,7 +33,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--profile', '-p', default='.chrome_profile')
     parser.add_argument('--url', '-u', default='https://app.statdx.com/')
-    parser.add_argument('--domain', '-d', default='app.statdx.com', help='Domain to filter captures')
+    parser.add_argument('--domain', '-d', default='', help='Domain to filter captures')
     parser.add_argument('--screenshot', '-s', default='chrome_page.png')
     parser.add_argument('--wait-for', help='Optional selector to wait for before screenshot')
     parser.add_argument('--headless', action='store_true')
@@ -137,7 +137,17 @@ def main():
                     # Response body (text or binary)
                     ctype = resp.headers.get('content-type', '')
                     try:
-                        if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
+                        # Save HTML/document responses as separate .html files
+                        if 'text/html' in ctype or rtype == 'document':
+                            txt = resp.text()
+                            html_name = f"{sanitize_filename(req.url)}_{ts}.html"
+                            html_path = os.path.join(args.capture_output, html_name)
+                            with open(html_path, 'w', encoding='utf-8') as fh:
+                                fh.write(txt)
+                            info['response_text'] = None
+                            info['response_body_file'] = html_path
+                        elif 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
+                            # keep small textual responses inline in the metadata JSON
                             txt = resp.text()
                             info['response_text'] = txt
                             info['response_body_file'] = None