ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor logging to use Loguru in Playwright scripts for improved logging capabilities

Browse Source
This commit is contained in:
Ross
2025-10-14 14:09:19 +01:00
parent fd5922b3a6
commit 448db2aaf9
2 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/statdx_play_chrome.py
+20 -20
View File
@@ -13,7 +13,7 @@ Notes:
"""
import argparse
import logging
from loguru import logger
import os
from time import sleep
from urllib.parse import urlparse
@@ -22,12 +22,11 @@ import mimetypes
from datetime import datetime, timezone
from playwright.sync_api import sync_playwright
from dotenv import load_dotenv
from loguru import logger
# Load environment variables from .env in the repository root (if present)
load_dotenv()
from loguru import logger
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
def main():
@@ -46,17 +45,18 @@ def main():
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
parser.add_argument('--capture-types', default='xhr,fetch,document', help='Comma-separated list of resource types to capture (e.g. xhr,fetch,document,image)')
args = parser.parse_args()
os.makedirs(args.profile, exist_ok=True)
with sync_playwright() as p:
try:
logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
logger.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
except Exception as e:
logging.error('Failed to launch Chrome channel: %s', e)
logging.info('Falling back to default Chromium persistent context')
logger.error(f'Failed to launch Chrome channel: {e}')
logger.info('Falling back to default Chromium persistent context')
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
page = context.new_page()
@@ -95,7 +95,7 @@ def main():
def on_response(resp):
# lightweight debug log
logging.debug('Response: %s %s', resp.status, resp.url)
logger.debug(f'Response: {resp.status} {resp.url}')
try:
req = resp.request
rtype = req.resource_type
@@ -145,7 +145,7 @@ def main():
info['response_text'] = None
info['response_body_file'] = path
except Exception:
logging.exception('Failed to read response body for %s', req.url)
logger.exception(f'Failed to read response body for {req.url}')
fname = f"{sanitize_filename(req.url)}_{ts}.json"
out_path = os.path.join(args.capture_output, fname)
@@ -154,10 +154,10 @@ def main():
json.dump(info, fh, indent=2)
logging.info('Captured %s: %s -> %s', rtype, req.url, out_path)
logger.info(f'Captured {rtype}: {req.url} -> {out_path}')
captured.append(out_path)
except Exception:
logging.exception('Error in capture on_response')
logger.exception('Error in capture on_response')
page.on('response', on_response)
@@ -165,7 +165,7 @@ def main():
username = args.username or os.getenv('STATDX_USERNAME')
password = args.password or os.getenv('STATDX_PASSWORD')
if username and password:
logging.info('Attempting automatic login using provided credentials')
logger.info('Attempting automatic login using provided credentials')
# candidate selectors for username and password fields
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
@@ -189,7 +189,7 @@ def main():
continue
if not user_sel or not pass_sel:
logging.warning('Could not auto-detect username/password fields. Skipping automatic login.')
logger.warning('Could not auto-detect username/password fields. Skipping automatic login.')
else:
try:
page.fill(user_sel, username)
@@ -205,12 +205,12 @@ def main():
if btn:
btn.click()
clicked = True
logging.info('Clicked submit button using selector: %s', s)
logger.info(f'Clicked submit button using selector: {s}')
break
except Exception:
continue
if not clicked:
logging.info('No submit button clicked; pressing Enter in password field')
logger.info('No submit button clicked; pressing Enter in password field')
page.press(pass_sel, 'Enter')
# wait for either navigation or a post-login selector
@@ -218,19 +218,19 @@ def main():
if args.post_login_selector:
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
except Exception:
logging.info('Post-login selector not found within timeout')
logger.info('Post-login selector not found within timeout')
# small extra wait to let JS render
sleep(args.wait_after_login)
except Exception:
logging.exception('Automatic login attempt failed')
logger.exception('Automatic login attempt failed')
if args.wait_for:
try:
logging.info('Waiting for selector: %s', args.wait_for)
logger.info(f'Waiting for selector: {args.wait_for}')
page.wait_for_selector(args.wait_for, timeout=20000)
except Exception:
logging.exception('wait_for selector timed out')
logger.exception('wait_for selector timed out')
# give some time for JS to finish
page.wait_for_timeout(1500)
@@ -238,7 +238,7 @@ def main():
# Screenshots are disabled per user request
if args.keep_open:
logging.info('Leaving browser open for manual inspection. Close the browser window to finish.')
logger.info('Leaving browser open for manual inspection. Close the browser window to finish.')
try:
# keep running until the user presses Enter in the terminal
input('Browser is open. Press Enter here to close and exit...')
@@ -248,7 +248,7 @@ def main():
try:
context.close()
except Exception:
logging.exception('Error closing context')
logger.exception('Error closing context')
if __name__ == '__main__':