Add Playwright-based scraping and login scripts for STATdx
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
This commit is contained in:
Ross
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Form-based login + scraping example for STATdx (best-effort).
|
||||
|
||||
This script attempts to:
|
||||
- GET the login page
|
||||
- detect a login form and its fields (username/password)
|
||||
- submit credentials and preserve the session cookie
|
||||
- fetch the target topics page and print/save HTML
|
||||
|
||||
Notes:
|
||||
- Most modern sites use JS-driven auth or SSO; if this fails, use the Playwright script.
|
||||
- Inspect the login POST URL and field names with your browser and set the environment overrides in .env
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
# Configuration via environment (or fallbacks)
|
||||
LOGIN_PAGE = os.getenv("STATDX_LOGIN_URL", "https://app.statdx.com/")
|
||||
TOPICS_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
||||
USERNAME = os.getenv("STATDX_USERNAME")
|
||||
PASSWORD = os.getenv("STATDX_PASSWORD")
|
||||
OVERRIDE_USER_FIELD = os.getenv("STATDX_USERNAME_FIELD")
|
||||
OVERRIDE_PASS_FIELD = os.getenv("STATDX_PASSWORD_FIELD")
|
||||
|
||||
if not USERNAME or not PASSWORD:
|
||||
logging.error("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
||||
sys.exit(1)
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
def find_login_form(html, base_url):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
forms = soup.find_all("form")
|
||||
if not forms:
|
||||
return None
|
||||
|
||||
# Heuristic: pick the form that contains an input of type password
|
||||
for form in forms:
|
||||
if form.find("input", {"type": "password"}):
|
||||
action = form.get("action") or base_url
|
||||
method = (form.get("method") or "get").lower()
|
||||
inputs = {}
|
||||
for inp in form.find_all("input"):
|
||||
name = inp.get("name")
|
||||
if not name:
|
||||
continue
|
||||
value = inp.get("value", "")
|
||||
inputs[name] = value
|
||||
return {"action": urljoin(base_url, action), "method": method, "inputs": inputs}
|
||||
return None
|
||||
|
||||
|
||||
def guess_field_names(inputs):
|
||||
# Try to find which input names likely correspond to username and password
|
||||
u_name = None
|
||||
p_name = None
|
||||
for name in inputs:
|
||||
ln = name.lower()
|
||||
if any(x in ln for x in ("user", "email", "login")) and not u_name:
|
||||
u_name = name
|
||||
if "pass" in ln and not p_name:
|
||||
p_name = name
|
||||
# Fallback to common names
|
||||
if not u_name:
|
||||
for cand in ("username", "email", "user"):
|
||||
if cand in inputs:
|
||||
u_name = cand
|
||||
break
|
||||
if not p_name:
|
||||
for cand in ("password", "pass"):
|
||||
if cand in inputs:
|
||||
p_name = cand
|
||||
break
|
||||
return u_name, p_name
|
||||
|
||||
|
||||
def login_using_form(login_page_url):
|
||||
logging.info("Fetching login page: %s", login_page_url)
|
||||
r = session.get(login_page_url, allow_redirects=True)
|
||||
r.raise_for_status()
|
||||
form = find_login_form(r.text, r.url)
|
||||
if not form:
|
||||
logging.warning("No login form found on page. The site might be JS-driven. Consider the Playwright script.")
|
||||
return False
|
||||
|
||||
action = form["action"]
|
||||
method = form["method"]
|
||||
inputs = form["inputs"].copy()
|
||||
|
||||
# Determine username / password field names
|
||||
user_field = OVERRIDE_USER_FIELD or guess_field_names(inputs)[0]
|
||||
pass_field = OVERRIDE_PASS_FIELD or guess_field_names(inputs)[1]
|
||||
|
||||
if not user_field or not pass_field:
|
||||
logging.error("Could not detect username/password field names. Please inspect the login form and set STATDX_USERNAME_FIELD and STATDX_PASSWORD_FIELD in .env")
|
||||
return False
|
||||
|
||||
inputs[user_field] = USERNAME
|
||||
inputs[pass_field] = PASSWORD
|
||||
|
||||
logging.info("Submitting login form to: %s", action)
|
||||
if method == "post":
|
||||
resp = session.post(action, data=inputs, allow_redirects=True)
|
||||
else:
|
||||
resp = session.get(action, params=inputs, allow_redirects=True)
|
||||
|
||||
logging.info("Login response: %s %s", resp.status_code, resp.url)
|
||||
|
||||
# Basic success check: did we land on a different URL or see a known element?
|
||||
if resp.url != login_page_url and resp.status_code in (200, 302, 303):
|
||||
logging.info("Looks like login may have succeeded (redirected).")
|
||||
return True
|
||||
|
||||
# Heuristic: search for 'logout' or 'sign out' in HTML
|
||||
if "logout" in resp.text.lower() or "sign out" in resp.text.lower():
|
||||
logging.info("Found logout text — login probably succeeded.")
|
||||
return True
|
||||
|
||||
logging.warning("Login may have failed (no redirect or logout text). Try Playwright or provide exact POST URL/fields.)")
|
||||
return False
|
||||
|
||||
|
||||
def fetch_topics(url):
|
||||
logging.info("Fetching topics page: %s", url)
|
||||
r = session.get(url)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
||||
def save_html(content, path="topics.html"):
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
logging.info("Saved HTML to %s", path)
|
||||
|
||||
|
||||
def main():
|
||||
ok = login_using_form(LOGIN_PAGE)
|
||||
if not ok:
|
||||
logging.error("Form-based login failed. If the site uses JS/SSO, try the Playwright script: scrapers/statdx_playwright.py")
|
||||
return
|
||||
|
||||
html = fetch_topics(TOPICS_URL)
|
||||
save_html(html)
|
||||
# As a minimal parser example, show the contents of <div id="ds-app">
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
ds_app = soup.find(id="ds-app")
|
||||
if ds_app:
|
||||
logging.info("Found #ds-app — page likely relies on JavaScript. For rendered content use Playwright to get client-side-rendered HTML.")
|
||||
else:
|
||||
logging.info("#ds-app not found (or empty). You can parse the HTML above for static content.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user