ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor logging to use Loguru in Playwright scripts for improved logging capabilities

Browse Source
This commit is contained in:
Ross
2025-10-14 14:09:19 +01:00
parent fd5922b3a6
commit 448db2aaf9
2 changed files with 34 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/statdx_playwright.py
+14 -14
View File
@@ -9,7 +9,7 @@ You may need to tweak selectors if the login form uses unusual markup or an SSO
import os
import time
import logging
from loguru import logger
from dotenv import load_dotenv
load_dotenv()
@@ -21,7 +21,7 @@ TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
if not USERNAME or not PASSWORD:
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
# using loguru logger
def run(headless=True, timeout=30000):
@@ -32,16 +32,16 @@ def run(headless=True, timeout=30000):
context = browser.new_context()
page = context.new_page()
logging.info("Opening %s", TARGET_URL)
logger.info("Opening %s", TARGET_URL)
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
# Heuristics to find fields — adapt per the real login form
password_selector = "input[type=\"password\"]"
try:
page.wait_for_selector(password_selector, timeout=8000)
logging.info("Password field found")
logger.info("Password field found")
except Exception:
logging.warning("Password field not found quickly; continuing to try common selectors")
logger.warning("Password field not found quickly; continuing to try common selectors")
# Try to detect username/email field by common selectors
candidate_user_selectors = [
@@ -61,18 +61,18 @@ def run(headless=True, timeout=30000):
continue
if not user_sel:
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
logger.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
# fallback: pick first visible text input
inputs = page.query_selector_all("input[type=\"text\"]")
if inputs:
user_sel = "input[type=\"text\"]"
logging.info("Using username selector: %s", user_sel)
logger.info("Using username selector: %s", user_sel)
if user_sel:
page.fill(user_sel, USERNAME)
else:
logging.error("No username field found. Aborting.")
logger.error("No username field found. Aborting.")
browser.close()
return
@@ -80,7 +80,7 @@ def run(headless=True, timeout=30000):
if page.query_selector(password_selector):
page.fill(password_selector, PASSWORD)
else:
logging.error("No password field found. Aborting.")
logger.error("No password field found. Aborting.")
browser.close()
return
@@ -101,21 +101,21 @@ def run(headless=True, timeout=30000):
if btn:
btn.click()
clicked = True
logging.info("Clicked submit via selector %s", s)
logger.info("Clicked submit via selector %s", s)
break
except Exception:
continue
if not clicked:
logging.warning("No submit button clicked; attempting to press Enter on password field")
logger.warning("No submit button clicked; attempting to press Enter on password field")
page.press(password_selector, "Enter")
# Wait for navigation or the app root element that indicates successful login
try:
page.wait_for_selector("#ds-app", timeout=20000)
logging.info("#ds-app found — likely logged in and app rendered")
logger.info("#ds-app found — likely logged in and app rendered")
except Exception:
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
logger.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
# Optional: wait a bit for client-side rendering
time.sleep(2)
@@ -124,7 +124,7 @@ def run(headless=True, timeout=30000):
out_file = "topics_playwright.html"
with open(out_file, "w", encoding="utf-8") as f:
f.write(content)
logging.info("Saved rendered HTML to %s", out_file)
logger.info("Saved rendered HTML to %s", out_file)
browser.close()