From fd5922b3a6e7ac71015f559745aeb6b303ae4112 Mon Sep 17 00:00:00 2001
From: Ross <xkjq@ymail.com>
Date: Tue, 14 Oct 2025 13:56:47 +0100
Subject: [PATCH] Update .gitignore and requirements; enhance logging in
 Playwright scraper

---
 .gitignore                     |  3 ++-
 requirements.txt               |  1 +
 scrapers/statdx_play_chrome.py | 17 ++++++++++++-----
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 07a0fff..c570d10 100644
--- a/.gitignore
+++ b/.gitignore
@@ -67,4 +67,5 @@ secrets.json
 .chrome_profile
 .playwright_profile
 
-captured/
\ No newline at end of file
+captured/
+xhr_captured/
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index 0d4ed44..d285cc8 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@ requests>=2.28.0
 beautifulsoup4>=4.12.0
 playwright>=1.40.0
 python-dotenv>=1.0.0
+loguru
\ No newline at end of file
diff --git a/scrapers/statdx_play_chrome.py b/scrapers/statdx_play_chrome.py
index 1e3d546..621fc02 100644
--- a/scrapers/statdx_play_chrome.py
+++ b/scrapers/statdx_play_chrome.py
@@ -22,6 +22,7 @@ import mimetypes
 from datetime import datetime, timezone
 from playwright.sync_api import sync_playwright
 from dotenv import load_dotenv
+from loguru import logger
 
 # Load environment variables from .env in the repository root (if present)
 load_dotenv()
@@ -90,13 +91,17 @@ def main():
                     fh.write(body)
                 return path
 
+            capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()}
+
             def on_response(resp):
+                # lightweight debug log
+                logging.debug('Response: %s %s', resp.status, resp.url)
                 try:
                     req = resp.request
                     rtype = req.resource_type
                     if args.domain not in req.url:
                         return
-                    if rtype not in ('xhr', 'fetch'):
+                    if rtype not in capture_types:
                         return
 
                     ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
@@ -111,6 +116,7 @@ def main():
                         'timestamp': ts,
                     }
 
+                    # Request post data
                     try:
                         post = req.post_data
                         info['request_post_data'] = post
@@ -120,9 +126,10 @@ def main():
                         except Exception:
                             info['request_post_data'] = None
 
+                    # Response body (text or binary)
                     ctype = resp.headers.get('content-type', '')
                     try:
-                        if 'application/json' in ctype or 'text/' in ctype or 'application/javascript' in ctype:
+                        if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype:
                             txt = resp.text()
                             info['response_text'] = txt
                             info['response_body_file'] = None
@@ -147,7 +154,7 @@ def main():
 
                         json.dump(info, fh, indent=2)
 
-                    logging.info('Captured XHR: %s -> %s', req.url, out_path)
+                    logging.info('Captured %s: %s -> %s', rtype, req.url, out_path)
                     captured.append(out_path)
                 except Exception:
                     logging.exception('Error in capture on_response')
@@ -233,8 +240,8 @@ def main():
         if args.keep_open:
             logging.info('Leaving browser open for manual inspection. Close the browser window to finish.')
             try:
-                # keep running until user closes browser
-                context.wait_for_event('close')
+                # keep running until the user presses Enter in the terminal
+                input('Browser is open. Press Enter here to close and exit...')
             except Exception:
                 pass