Ross
491fca2f42
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
134 lines
4.5 KiB
Python
134 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Playwright-based login + scraping for STATdx (recommended for JS-heavy sites).
|
|
|
|
This script launches a headless Chromium, navigates to the app, fills username/password, clicks the submit button,
|
|
waits for a post-login selector, then saves the resulting HTML to disk.
|
|
|
|
You may need to tweak selectors if the login form uses unusual markup or an SSO redirect.
|
|
"""
|
|
|
|
import os
|
|
import time
|
|
import logging
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
USERNAME = os.getenv("STATDX_USERNAME")
|
|
PASSWORD = os.getenv("STATDX_PASSWORD")
|
|
TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
|
|
|
if not USERNAME or not PASSWORD:
|
|
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
|
|
|
|
|
def run(headless=True, timeout=30000):
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=headless)
|
|
context = browser.new_context()
|
|
page = context.new_page()
|
|
|
|
logging.info("Opening %s", TARGET_URL)
|
|
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
|
|
|
|
# Heuristics to find fields — adapt per the real login form
|
|
password_selector = "input[type=\"password\"]"
|
|
try:
|
|
page.wait_for_selector(password_selector, timeout=8000)
|
|
logging.info("Password field found")
|
|
except Exception:
|
|
logging.warning("Password field not found quickly; continuing to try common selectors")
|
|
|
|
# Try to detect username/email field by common selectors
|
|
candidate_user_selectors = [
|
|
"input[type=\"email\"]",
|
|
"input[name*=\"email\" i]",
|
|
"input[name*=\"user\" i]",
|
|
"input[type=\"text\"]",
|
|
]
|
|
|
|
user_sel = None
|
|
for sel in candidate_user_selectors:
|
|
try:
|
|
if page.query_selector(sel):
|
|
user_sel = sel
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not user_sel:
|
|
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
|
|
# fallback: pick first visible text input
|
|
inputs = page.query_selector_all("input[type=\"text\"]")
|
|
if inputs:
|
|
user_sel = "input[type=\"text\"]"
|
|
|
|
logging.info("Using username selector: %s", user_sel)
|
|
|
|
if user_sel:
|
|
page.fill(user_sel, USERNAME)
|
|
else:
|
|
logging.error("No username field found. Aborting.")
|
|
browser.close()
|
|
return
|
|
|
|
# Fill password
|
|
if page.query_selector(password_selector):
|
|
page.fill(password_selector, PASSWORD)
|
|
else:
|
|
logging.error("No password field found. Aborting.")
|
|
browser.close()
|
|
return
|
|
|
|
# Try to click a submit button
|
|
submit_selectors = [
|
|
"button[type=submit]",
|
|
"input[type=submit]",
|
|
"button:has-text(\"Sign in\")",
|
|
"button:has-text(\"Sign In\")",
|
|
"button:has-text(\"Log in\")",
|
|
"button:has-text(\"Log In\")",
|
|
]
|
|
|
|
clicked = False
|
|
for s in submit_selectors:
|
|
try:
|
|
btn = page.query_selector(s)
|
|
if btn:
|
|
btn.click()
|
|
clicked = True
|
|
logging.info("Clicked submit via selector %s", s)
|
|
break
|
|
except Exception:
|
|
continue
|
|
|
|
if not clicked:
|
|
logging.warning("No submit button clicked; attempting to press Enter on password field")
|
|
page.press(password_selector, "Enter")
|
|
|
|
# Wait for navigation or the app root element that indicates successful login
|
|
try:
|
|
page.wait_for_selector("#ds-app", timeout=20000)
|
|
logging.info("#ds-app found — likely logged in and app rendered")
|
|
except Exception:
|
|
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
|
|
|
|
# Optional: wait a bit for client-side rendering
|
|
time.sleep(2)
|
|
|
|
content = page.content()
|
|
out_file = "topics_playwright.html"
|
|
with open(out_file, "w", encoding="utf-8") as f:
|
|
f.write(content)
|
|
logging.info("Saved rendered HTML to %s", out_file)
|
|
|
|
browser.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(headless=True)
|