diff --git a/scrapers/statdx_play_chrome.py b/scrapers/statdx_play_chrome.py index 621fc02..eb9e88a 100644 --- a/scrapers/statdx_play_chrome.py +++ b/scrapers/statdx_play_chrome.py @@ -13,7 +13,7 @@ Notes: """ import argparse -import logging +from loguru import logger import os from time import sleep from urllib.parse import urlparse @@ -22,12 +22,11 @@ import mimetypes from datetime import datetime, timezone from playwright.sync_api import sync_playwright from dotenv import load_dotenv -from loguru import logger # Load environment variables from .env in the repository root (if present) load_dotenv() +from loguru import logger -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') def main(): @@ -46,17 +45,18 @@ def main(): parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open') parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs') parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts') + parser.add_argument('--capture-types', default='xhr,fetch,document', help='Comma-separated list of resource types to capture (e.g. xhr,fetch,document,image)') args = parser.parse_args() os.makedirs(args.profile, exist_ok=True) with sync_playwright() as p: try: - logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless) + logger.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless) context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless) except Exception as e: - logging.error('Failed to launch Chrome channel: %s', e) - logging.info('Falling back to default Chromium persistent context') + logger.error(f'Failed to launch Chrome channel: {e}') + logger.info('Falling back to default Chromium persistent context') context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) page = context.new_page() @@ -95,7 +95,7 @@ def main(): def on_response(resp): # lightweight debug log - logging.debug('Response: %s %s', resp.status, resp.url) + logger.debug(f'Response: {resp.status} {resp.url}') try: req = resp.request rtype = req.resource_type @@ -145,7 +145,7 @@ def main(): info['response_text'] = None info['response_body_file'] = path except Exception: - logging.exception('Failed to read response body for %s', req.url) + logger.exception(f'Failed to read response body for {req.url}') fname = f"{sanitize_filename(req.url)}_{ts}.json" out_path = os.path.join(args.capture_output, fname) @@ -154,10 +154,10 @@ def main(): json.dump(info, fh, indent=2) - logging.info('Captured %s: %s -> %s', rtype, req.url, out_path) + logger.info(f'Captured {rtype}: {req.url} -> {out_path}') captured.append(out_path) except Exception: - logging.exception('Error in capture on_response') + logger.exception('Error in capture on_response') page.on('response', on_response) @@ -165,7 +165,7 @@ def main(): username = args.username or os.getenv('STATDX_USERNAME') password = args.password or os.getenv('STATDX_PASSWORD') if username and password: - logging.info('Attempting automatic login using provided credentials') + logger.info('Attempting automatic login using provided credentials') # candidate selectors for username and password fields user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]'] pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector'] @@ -189,7 +189,7 @@ def main(): continue if not user_sel or not pass_sel: - logging.warning('Could not auto-detect username/password fields. Skipping automatic login.') + logger.warning('Could not auto-detect username/password fields. Skipping automatic login.') else: try: page.fill(user_sel, username) @@ -205,12 +205,12 @@ def main(): if btn: btn.click() clicked = True - logging.info('Clicked submit button using selector: %s', s) + logger.info(f'Clicked submit button using selector: {s}') break except Exception: continue if not clicked: - logging.info('No submit button clicked; pressing Enter in password field') + logger.info('No submit button clicked; pressing Enter in password field') page.press(pass_sel, 'Enter') # wait for either navigation or a post-login selector @@ -218,19 +218,19 @@ def main(): if args.post_login_selector: page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000)) except Exception: - logging.info('Post-login selector not found within timeout') + logger.info('Post-login selector not found within timeout') # small extra wait to let JS render sleep(args.wait_after_login) except Exception: - logging.exception('Automatic login attempt failed') + logger.exception('Automatic login attempt failed') if args.wait_for: try: - logging.info('Waiting for selector: %s', args.wait_for) + logger.info(f'Waiting for selector: {args.wait_for}') page.wait_for_selector(args.wait_for, timeout=20000) except Exception: - logging.exception('wait_for selector timed out') + logger.exception('wait_for selector timed out') # give some time for JS to finish page.wait_for_timeout(1500) @@ -238,7 +238,7 @@ def main(): # Screenshots are disabled per user request if args.keep_open: - logging.info('Leaving browser open for manual inspection. Close the browser window to finish.') + logger.info('Leaving browser open for manual inspection. Close the browser window to finish.') try: # keep running until the user presses Enter in the terminal input('Browser is open. Press Enter here to close and exit...') @@ -248,7 +248,7 @@ def main(): try: context.close() except Exception: - logging.exception('Error closing context') + logger.exception('Error closing context') if __name__ == '__main__': diff --git a/scrapers/statdx_playwright.py b/scrapers/statdx_playwright.py index 8511b99..8fe430b 100644 --- a/scrapers/statdx_playwright.py +++ b/scrapers/statdx_playwright.py @@ -9,7 +9,7 @@ You may need to tweak selectors if the login form uses unusual markup or an SSO import os import time -import logging +from loguru import logger from dotenv import load_dotenv load_dotenv() @@ -21,7 +21,7 @@ TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/") if not USERNAME or not PASSWORD: raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.") -logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +# using loguru logger def run(headless=True, timeout=30000): @@ -32,16 +32,16 @@ def run(headless=True, timeout=30000): context = browser.new_context() page = context.new_page() - logging.info("Opening %s", TARGET_URL) + logger.info("Opening %s", TARGET_URL) page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout) # Heuristics to find fields — adapt per the real login form password_selector = "input[type=\"password\"]" try: page.wait_for_selector(password_selector, timeout=8000) - logging.info("Password field found") + logger.info("Password field found") except Exception: - logging.warning("Password field not found quickly; continuing to try common selectors") + logger.warning("Password field not found quickly; continuing to try common selectors") # Try to detect username/email field by common selectors candidate_user_selectors = [ @@ -61,18 +61,18 @@ def run(headless=True, timeout=30000): continue if not user_sel: - logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.") + logger.warning("Could not auto-detect username field; you may need to edit selectors in this script.") # fallback: pick first visible text input inputs = page.query_selector_all("input[type=\"text\"]") if inputs: user_sel = "input[type=\"text\"]" - logging.info("Using username selector: %s", user_sel) + logger.info("Using username selector: %s", user_sel) if user_sel: page.fill(user_sel, USERNAME) else: - logging.error("No username field found. Aborting.") + logger.error("No username field found. Aborting.") browser.close() return @@ -80,7 +80,7 @@ def run(headless=True, timeout=30000): if page.query_selector(password_selector): page.fill(password_selector, PASSWORD) else: - logging.error("No password field found. Aborting.") + logger.error("No password field found. Aborting.") browser.close() return @@ -101,21 +101,21 @@ def run(headless=True, timeout=30000): if btn: btn.click() clicked = True - logging.info("Clicked submit via selector %s", s) + logger.info("Clicked submit via selector %s", s) break except Exception: continue if not clicked: - logging.warning("No submit button clicked; attempting to press Enter on password field") + logger.warning("No submit button clicked; attempting to press Enter on password field") page.press(password_selector, "Enter") # Wait for navigation or the app root element that indicates successful login try: page.wait_for_selector("#ds-app", timeout=20000) - logging.info("#ds-app found — likely logged in and app rendered") + logger.info("#ds-app found — likely logged in and app rendered") except Exception: - logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.") + logger.info("Timed out waiting for #ds-app — page content may still be loading or login failed.") # Optional: wait a bit for client-side rendering time.sleep(2) @@ -124,7 +124,7 @@ def run(headless=True, timeout=30000): out_file = "topics_playwright.html" with open(out_file, "w", encoding="utf-8") as f: f.write(content) - logging.info("Saved rendered HTML to %s", out_file) + logger.info("Saved rendered HTML to %s", out_file) browser.close()