ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor logging to use Loguru in Playwright scripts for improved logging capabilities

Browse Source
This commit is contained in:
Ross
2025-10-14 14:09:19 +01:00
parent fd5922b3a6
commit 448db2aaf9
2 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/statdx_play_chrome.py
+20 -20
View File
@@ -13,7 +13,7 @@ Notes:
""" """
import argparse import argparse
import logging from loguru import logger
import os import os
from time import sleep from time import sleep
from urllib.parse import urlparse from urllib.parse import urlparse
@@ -22,12 +22,11 @@ import mimetypes
from datetime import datetime, timezone from datetime import datetime, timezone
from playwright.sync_api import sync_playwright from playwright.sync_api import sync_playwright
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger
# Load environment variables from .env in the repository root (if present) # Load environment variables from .env in the repository root (if present)
load_dotenv() load_dotenv()
from loguru import logger
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
def main(): def main():
@@ -46,17 +45,18 @@ def main():
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open') parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs') parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts') parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
parser.add_argument('--capture-types', default='xhr,fetch,document', help='Comma-separated list of resource types to capture (e.g. xhr,fetch,document,image)')
args = parser.parse_args() args = parser.parse_args()
os.makedirs(args.profile, exist_ok=True) os.makedirs(args.profile, exist_ok=True)
with sync_playwright() as p: with sync_playwright() as p:
try: try:
logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless) logger.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless) context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
except Exception as e: except Exception as e:
logging.error('Failed to launch Chrome channel: %s', e) logger.error(f'Failed to launch Chrome channel: {e}')
logging.info('Falling back to default Chromium persistent context') logger.info('Falling back to default Chromium persistent context')
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
page = context.new_page() page = context.new_page()
@@ -95,7 +95,7 @@ def main():
def on_response(resp): def on_response(resp):
# lightweight debug log # lightweight debug log
logging.debug('Response: %s %s', resp.status, resp.url) logger.debug(f'Response: {resp.status} {resp.url}')
try: try:
req = resp.request req = resp.request
rtype = req.resource_type rtype = req.resource_type
@@ -145,7 +145,7 @@ def main():
info['response_text'] = None info['response_text'] = None
info['response_body_file'] = path info['response_body_file'] = path
except Exception: except Exception:
logging.exception('Failed to read response body for %s', req.url) logger.exception(f'Failed to read response body for {req.url}')
fname = f"{sanitize_filename(req.url)}_{ts}.json" fname = f"{sanitize_filename(req.url)}_{ts}.json"
out_path = os.path.join(args.capture_output, fname) out_path = os.path.join(args.capture_output, fname)
@@ -154,10 +154,10 @@ def main():
json.dump(info, fh, indent=2) json.dump(info, fh, indent=2)
logging.info('Captured %s: %s -> %s', rtype, req.url, out_path) logger.info(f'Captured {rtype}: {req.url} -> {out_path}')
captured.append(out_path) captured.append(out_path)
except Exception: except Exception:
logging.exception('Error in capture on_response') logger.exception('Error in capture on_response')
page.on('response', on_response) page.on('response', on_response)
@@ -165,7 +165,7 @@ def main():
username = args.username or os.getenv('STATDX_USERNAME') username = args.username or os.getenv('STATDX_USERNAME')
password = args.password or os.getenv('STATDX_PASSWORD') password = args.password or os.getenv('STATDX_PASSWORD')
if username and password: if username and password:
logging.info('Attempting automatic login using provided credentials') logger.info('Attempting automatic login using provided credentials')
# candidate selectors for username and password fields # candidate selectors for username and password fields
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]'] user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector'] pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
@@ -189,7 +189,7 @@ def main():
continue continue
if not user_sel or not pass_sel: if not user_sel or not pass_sel:
logging.warning('Could not auto-detect username/password fields. Skipping automatic login.') logger.warning('Could not auto-detect username/password fields. Skipping automatic login.')
else: else:
try: try:
page.fill(user_sel, username) page.fill(user_sel, username)
@@ -205,12 +205,12 @@ def main():
if btn: if btn:
btn.click() btn.click()
clicked = True clicked = True
logging.info('Clicked submit button using selector: %s', s) logger.info(f'Clicked submit button using selector: {s}')
break break
except Exception: except Exception:
continue continue
if not clicked: if not clicked:
logging.info('No submit button clicked; pressing Enter in password field') logger.info('No submit button clicked; pressing Enter in password field')
page.press(pass_sel, 'Enter') page.press(pass_sel, 'Enter')
# wait for either navigation or a post-login selector # wait for either navigation or a post-login selector
@@ -218,19 +218,19 @@ def main():
if args.post_login_selector: if args.post_login_selector:
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000)) page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
except Exception: except Exception:
logging.info('Post-login selector not found within timeout') logger.info('Post-login selector not found within timeout')
# small extra wait to let JS render # small extra wait to let JS render
sleep(args.wait_after_login) sleep(args.wait_after_login)
except Exception: except Exception:
logging.exception('Automatic login attempt failed') logger.exception('Automatic login attempt failed')
if args.wait_for: if args.wait_for:
try: try:
logging.info('Waiting for selector: %s', args.wait_for) logger.info(f'Waiting for selector: {args.wait_for}')
page.wait_for_selector(args.wait_for, timeout=20000) page.wait_for_selector(args.wait_for, timeout=20000)
except Exception: except Exception:
logging.exception('wait_for selector timed out') logger.exception('wait_for selector timed out')
# give some time for JS to finish # give some time for JS to finish
page.wait_for_timeout(1500) page.wait_for_timeout(1500)
@@ -238,7 +238,7 @@ def main():
# Screenshots are disabled per user request # Screenshots are disabled per user request
if args.keep_open: if args.keep_open:
logging.info('Leaving browser open for manual inspection. Close the browser window to finish.') logger.info('Leaving browser open for manual inspection. Close the browser window to finish.')
try: try:
# keep running until the user presses Enter in the terminal # keep running until the user presses Enter in the terminal
input('Browser is open. Press Enter here to close and exit...') input('Browser is open. Press Enter here to close and exit...')
@@ -248,7 +248,7 @@ def main():
try: try:
context.close() context.close()
except Exception: except Exception:
logging.exception('Error closing context') logger.exception('Error closing context')
if __name__ == '__main__': if __name__ == '__main__':
scrapers/statdx_playwright.py
+14 -14
View File
@@ -9,7 +9,7 @@ You may need to tweak selectors if the login form uses unusual markup or an SSO
import os import os
import time import time
import logging from loguru import logger
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
@@ -21,7 +21,7 @@ TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
if not USERNAME or not PASSWORD: if not USERNAME or not PASSWORD:
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.") raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") # using loguru logger
def run(headless=True, timeout=30000): def run(headless=True, timeout=30000):
@@ -32,16 +32,16 @@ def run(headless=True, timeout=30000):
context = browser.new_context() context = browser.new_context()
page = context.new_page() page = context.new_page()
logging.info("Opening %s", TARGET_URL) logger.info("Opening %s", TARGET_URL)
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout) page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
# Heuristics to find fields — adapt per the real login form # Heuristics to find fields — adapt per the real login form
password_selector = "input[type=\"password\"]" password_selector = "input[type=\"password\"]"
try: try:
page.wait_for_selector(password_selector, timeout=8000) page.wait_for_selector(password_selector, timeout=8000)
logging.info("Password field found") logger.info("Password field found")
except Exception: except Exception:
logging.warning("Password field not found quickly; continuing to try common selectors") logger.warning("Password field not found quickly; continuing to try common selectors")
# Try to detect username/email field by common selectors # Try to detect username/email field by common selectors
candidate_user_selectors = [ candidate_user_selectors = [
@@ -61,18 +61,18 @@ def run(headless=True, timeout=30000):
continue continue
if not user_sel: if not user_sel:
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.") logger.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
# fallback: pick first visible text input # fallback: pick first visible text input
inputs = page.query_selector_all("input[type=\"text\"]") inputs = page.query_selector_all("input[type=\"text\"]")
if inputs: if inputs:
user_sel = "input[type=\"text\"]" user_sel = "input[type=\"text\"]"
logging.info("Using username selector: %s", user_sel) logger.info("Using username selector: %s", user_sel)
if user_sel: if user_sel:
page.fill(user_sel, USERNAME) page.fill(user_sel, USERNAME)
else: else:
logging.error("No username field found. Aborting.") logger.error("No username field found. Aborting.")
browser.close() browser.close()
return return
@@ -80,7 +80,7 @@ def run(headless=True, timeout=30000):
if page.query_selector(password_selector): if page.query_selector(password_selector):
page.fill(password_selector, PASSWORD) page.fill(password_selector, PASSWORD)
else: else:
logging.error("No password field found. Aborting.") logger.error("No password field found. Aborting.")
browser.close() browser.close()
return return
@@ -101,21 +101,21 @@ def run(headless=True, timeout=30000):
if btn: if btn:
btn.click() btn.click()
clicked = True clicked = True
logging.info("Clicked submit via selector %s", s) logger.info("Clicked submit via selector %s", s)
break break
except Exception: except Exception:
continue continue
if not clicked: if not clicked:
logging.warning("No submit button clicked; attempting to press Enter on password field") logger.warning("No submit button clicked; attempting to press Enter on password field")
page.press(password_selector, "Enter") page.press(password_selector, "Enter")
# Wait for navigation or the app root element that indicates successful login # Wait for navigation or the app root element that indicates successful login
try: try:
page.wait_for_selector("#ds-app", timeout=20000) page.wait_for_selector("#ds-app", timeout=20000)
logging.info("#ds-app found — likely logged in and app rendered") logger.info("#ds-app found — likely logged in and app rendered")
except Exception: except Exception:
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.") logger.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
# Optional: wait a bit for client-side rendering # Optional: wait a bit for client-side rendering
time.sleep(2) time.sleep(2)
@@ -124,7 +124,7 @@ def run(headless=True, timeout=30000):
out_file = "topics_playwright.html" out_file = "topics_playwright.html"
with open(out_file, "w", encoding="utf-8") as f: with open(out_file, "w", encoding="utf-8") as f:
f.write(content) f.write(content)
logging.info("Saved rendered HTML to %s", out_file) logger.info("Saved rendered HTML to %s", out_file)
browser.close() browser.close()