ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor logging to use Loguru in Playwright scripts for improved logging capabilities

Browse Source
This commit is contained in:
Ross
2025-10-14 14:09:19 +01:00
parent fd5922b3a6
commit 448db2aaf9
2 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/statdx_play_chrome.py
+20 -20
View File
@@ -13,7 +13,7 @@ Notes:
"""
import argparse
import logging
from loguru import logger
import os
from time import sleep
from urllib.parse import urlparse
@@ -22,12 +22,11 @@ import mimetypes
from datetime import datetime, timezone
from playwright.sync_api import sync_playwright
from dotenv import load_dotenv
from loguru import logger
# Load environment variables from .env in the repository root (if present)
load_dotenv()
from loguru import logger
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
def main():
@@ -46,17 +45,18 @@ def main():
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
parser.add_argument('--capture-types', default='xhr,fetch,document', help='Comma-separated list of resource types to capture (e.g. xhr,fetch,document,image)')
args = parser.parse_args()
os.makedirs(args.profile, exist_ok=True)
with sync_playwright() as p:
try:
logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
logger.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
except Exception as e:
logging.error('Failed to launch Chrome channel: %s', e)
logging.info('Falling back to default Chromium persistent context')
logger.error(f'Failed to launch Chrome channel: {e}')
logger.info('Falling back to default Chromium persistent context')
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
page = context.new_page()
@@ -95,7 +95,7 @@ def main():
def on_response(resp):
# lightweight debug log
logging.debug('Response: %s %s', resp.status, resp.url)
logger.debug(f'Response: {resp.status} {resp.url}')
try:
req = resp.request
rtype = req.resource_type
@@ -145,7 +145,7 @@ def main():
info['response_text'] = None
info['response_body_file'] = path
except Exception:
logging.exception('Failed to read response body for %s', req.url)
logger.exception(f'Failed to read response body for {req.url}')
fname = f"{sanitize_filename(req.url)}_{ts}.json"
out_path = os.path.join(args.capture_output, fname)
@@ -154,10 +154,10 @@ def main():
json.dump(info, fh, indent=2)
logging.info('Captured %s: %s -> %s', rtype, req.url, out_path)
logger.info(f'Captured {rtype}: {req.url} -> {out_path}')
captured.append(out_path)
except Exception:
logging.exception('Error in capture on_response')
logger.exception('Error in capture on_response')
page.on('response', on_response)
@@ -165,7 +165,7 @@ def main():
username = args.username or os.getenv('STATDX_USERNAME')
password = args.password or os.getenv('STATDX_PASSWORD')
if username and password:
logging.info('Attempting automatic login using provided credentials')
logger.info('Attempting automatic login using provided credentials')
# candidate selectors for username and password fields
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
@@ -189,7 +189,7 @@ def main():
continue
if not user_sel or not pass_sel:
logging.warning('Could not auto-detect username/password fields. Skipping automatic login.')
logger.warning('Could not auto-detect username/password fields. Skipping automatic login.')
else:
try:
page.fill(user_sel, username)
@@ -205,12 +205,12 @@ def main():
if btn:
btn.click()
clicked = True
logging.info('Clicked submit button using selector: %s', s)
logger.info(f'Clicked submit button using selector: {s}')
break
except Exception:
continue
if not clicked:
logging.info('No submit button clicked; pressing Enter in password field')
logger.info('No submit button clicked; pressing Enter in password field')
page.press(pass_sel, 'Enter')
# wait for either navigation or a post-login selector
@@ -218,19 +218,19 @@ def main():
if args.post_login_selector:
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
except Exception:
logging.info('Post-login selector not found within timeout')
logger.info('Post-login selector not found within timeout')
# small extra wait to let JS render
sleep(args.wait_after_login)
except Exception:
logging.exception('Automatic login attempt failed')
logger.exception('Automatic login attempt failed')
if args.wait_for:
try:
logging.info('Waiting for selector: %s', args.wait_for)
logger.info(f'Waiting for selector: {args.wait_for}')
page.wait_for_selector(args.wait_for, timeout=20000)
except Exception:
logging.exception('wait_for selector timed out')
logger.exception('wait_for selector timed out')
# give some time for JS to finish
page.wait_for_timeout(1500)
@@ -238,7 +238,7 @@ def main():
# Screenshots are disabled per user request
if args.keep_open:
logging.info('Leaving browser open for manual inspection. Close the browser window to finish.')
logger.info('Leaving browser open for manual inspection. Close the browser window to finish.')
try:
# keep running until the user presses Enter in the terminal
input('Browser is open. Press Enter here to close and exit...')
@@ -248,7 +248,7 @@ def main():
try:
context.close()
except Exception:
logging.exception('Error closing context')
logger.exception('Error closing context')
if __name__ == '__main__':
scrapers/statdx_playwright.py
+14 -14
View File
@@ -9,7 +9,7 @@ You may need to tweak selectors if the login form uses unusual markup or an SSO
import os
import time
import logging
from loguru import logger
from dotenv import load_dotenv
load_dotenv()
@@ -21,7 +21,7 @@ TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
if not USERNAME or not PASSWORD:
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
# using loguru logger
def run(headless=True, timeout=30000):
@@ -32,16 +32,16 @@ def run(headless=True, timeout=30000):
context = browser.new_context()
page = context.new_page()
logging.info("Opening %s", TARGET_URL)
logger.info("Opening %s", TARGET_URL)
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
# Heuristics to find fields — adapt per the real login form
password_selector = "input[type=\"password\"]"
try:
page.wait_for_selector(password_selector, timeout=8000)
logging.info("Password field found")
logger.info("Password field found")
except Exception:
logging.warning("Password field not found quickly; continuing to try common selectors")
logger.warning("Password field not found quickly; continuing to try common selectors")
# Try to detect username/email field by common selectors
candidate_user_selectors = [
@@ -61,18 +61,18 @@ def run(headless=True, timeout=30000):
continue
if not user_sel:
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
logger.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
# fallback: pick first visible text input
inputs = page.query_selector_all("input[type=\"text\"]")
if inputs:
user_sel = "input[type=\"text\"]"
logging.info("Using username selector: %s", user_sel)
logger.info("Using username selector: %s", user_sel)
if user_sel:
page.fill(user_sel, USERNAME)
else:
logging.error("No username field found. Aborting.")
logger.error("No username field found. Aborting.")
browser.close()
return
@@ -80,7 +80,7 @@ def run(headless=True, timeout=30000):
if page.query_selector(password_selector):
page.fill(password_selector, PASSWORD)
else:
logging.error("No password field found. Aborting.")
logger.error("No password field found. Aborting.")
browser.close()
return
@@ -101,21 +101,21 @@ def run(headless=True, timeout=30000):
if btn:
btn.click()
clicked = True
logging.info("Clicked submit via selector %s", s)
logger.info("Clicked submit via selector %s", s)
break
except Exception:
continue
if not clicked:
logging.warning("No submit button clicked; attempting to press Enter on password field")
logger.warning("No submit button clicked; attempting to press Enter on password field")
page.press(password_selector, "Enter")
# Wait for navigation or the app root element that indicates successful login
try:
page.wait_for_selector("#ds-app", timeout=20000)
logging.info("#ds-app found — likely logged in and app rendered")
logger.info("#ds-app found — likely logged in and app rendered")
except Exception:
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
logger.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
# Optional: wait a bit for client-side rendering
time.sleep(2)
@@ -124,7 +124,7 @@ def run(headless=True, timeout=30000):
out_file = "topics_playwright.html"
with open(out_file, "w", encoding="utf-8") as f:
f.write(content)
logging.info("Saved rendered HTML to %s", out_file)
logger.info("Saved rendered HTML to %s", out_file)
browser.close()