statdx/scrapers/statdx_requests.py

#!/usr/bin/env python3
"""Form-based login + scraping example for STATdx (best-effort).

This script attempts to:
- GET the login page
- detect a login form and its fields (username/password)
- submit credentials and preserve the session cookie
- fetch the target topics page and print/save HTML

Notes:
- Most modern sites use JS-driven auth or SSO; if this fails, use the Playwright script.
- Inspect the login POST URL and field names with your browser and set the environment overrides in .env
"""

import os
import sys
import logging
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup
from dotenv import load_dotenv

load_dotenv()

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

# Configuration via environment (or fallbacks)
LOGIN_PAGE = os.getenv("STATDX_LOGIN_URL", "https://app.statdx.com/")
TOPICS_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
USERNAME = os.getenv("STATDX_USERNAME")
PASSWORD = os.getenv("STATDX_PASSWORD")
OVERRIDE_USER_FIELD = os.getenv("STATDX_USERNAME_FIELD")
OVERRIDE_PASS_FIELD = os.getenv("STATDX_PASSWORD_FIELD")

if not USERNAME or not PASSWORD:
    logging.error("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
    sys.exit(1)

session = requests.Session()


def find_login_form(html, base_url):
    soup = BeautifulSoup(html, "html.parser")
    forms = soup.find_all("form")
    if not forms:
        return None

    # Heuristic: pick the form that contains an input of type password
    for form in forms:
        if form.find("input", {"type": "password"}):
            action = form.get("action") or base_url
            method = (form.get("method") or "get").lower()
            inputs = {}
            for inp in form.find_all("input"):
                name = inp.get("name")
                if not name:
                    continue
                value = inp.get("value", "")
                inputs[name] = value
            return {"action": urljoin(base_url, action), "method": method, "inputs": inputs}
    return None


def guess_field_names(inputs):
    # Try to find which input names likely correspond to username and password
    u_name = None
    p_name = None
    for name in inputs:
        ln = name.lower()
        if any(x in ln for x in ("user", "email", "login")) and not u_name:
            u_name = name
        if "pass" in ln and not p_name:
            p_name = name
    # Fallback to common names
    if not u_name:
        for cand in ("username", "email", "user"):
            if cand in inputs:
                u_name = cand
                break
    if not p_name:
        for cand in ("password", "pass"):
            if cand in inputs:
                p_name = cand
                break
    return u_name, p_name


def login_using_form(login_page_url):
    logging.info("Fetching login page: %s", login_page_url)
    r = session.get(login_page_url, allow_redirects=True)
    r.raise_for_status()
    form = find_login_form(r.text, r.url)
    if not form:
        logging.warning("No login form found on page. The site might be JS-driven. Consider the Playwright script.")
        return False

    action = form["action"]
    method = form["method"]
    inputs = form["inputs"].copy()

    # Determine username / password field names
    user_field = OVERRIDE_USER_FIELD or guess_field_names(inputs)[0]
    pass_field = OVERRIDE_PASS_FIELD or guess_field_names(inputs)[1]

    if not user_field or not pass_field:
        logging.error("Could not detect username/password field names. Please inspect the login form and set STATDX_USERNAME_FIELD and STATDX_PASSWORD_FIELD in .env")
        return False

    inputs[user_field] = USERNAME
    inputs[pass_field] = PASSWORD

    logging.info("Submitting login form to: %s", action)
    if method == "post":
        resp = session.post(action, data=inputs, allow_redirects=True)
    else:
        resp = session.get(action, params=inputs, allow_redirects=True)

    logging.info("Login response: %s %s", resp.status_code, resp.url)

    # Basic success check: did we land on a different URL or see a known element?
    if resp.url != login_page_url and resp.status_code in (200, 302, 303):
        logging.info("Looks like login may have succeeded (redirected).")
        return True

    # Heuristic: search for 'logout' or 'sign out' in HTML
    if "logout" in resp.text.lower() or "sign out" in resp.text.lower():
        logging.info("Found logout text — login probably succeeded.")
        return True

    logging.warning("Login may have failed (no redirect or logout text). Try Playwright or provide exact POST URL/fields.)")
    return False


def fetch_topics(url):
    logging.info("Fetching topics page: %s", url)
    r = session.get(url)
    r.raise_for_status()
    return r.text


def save_html(content, path="topics.html"):
    with open(path, "w", encoding="utf-8") as f:
        f.write(content)
    logging.info("Saved HTML to %s", path)


def main():
    ok = login_using_form(LOGIN_PAGE)
    if not ok:
        logging.error("Form-based login failed. If the site uses JS/SSO, try the Playwright script: scrapers/statdx_playwright.py")
        return

    html = fetch_topics(TOPICS_URL)
    save_html(html)
    # As a minimal parser example, show the contents of <div id="ds-app">
    soup = BeautifulSoup(html, "html.parser")
    ds_app = soup.find(id="ds-app")
    if ds_app:
        logging.info("Found #ds-app — page likely relies on JavaScript. For rendered content use Playwright to get client-side-rendered HTML.")
    else:
        logging.info("#ds-app not found (or empty). You can parse the HTML above for static content.")


if __name__ == "__main__":
    main()