Add Playwright-based scraping and login scripts for STATdx
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
This commit is contained in:
Ross
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan captured JSON files and extract likely title/name fields.
|
||||
|
||||
Usage:
|
||||
python scrapers/parse_captured_json.py --dir captured --out parsed_topics.json
|
||||
|
||||
This prints a brief summary to stdout and writes a JSON file with discovered records.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
from glob import glob
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
LIKELY_KEYS = ['title', 'name', 'label', 'displayName', 'heading']
|
||||
|
||||
|
||||
def extract_from_obj(obj):
|
||||
found = []
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in LIKELY_KEYS and isinstance(v, (str, int, float)):
|
||||
found.append({'key': k, 'value': str(v)})
|
||||
else:
|
||||
found.extend(extract_from_obj(v))
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
found.extend(extract_from_obj(item))
|
||||
return found
|
||||
|
||||
|
||||
def process_file(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as fh:
|
||||
data = json.load(fh)
|
||||
except Exception:
|
||||
# skip files that aren't valid JSON
|
||||
return []
|
||||
return extract_from_obj(data)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--dir', '-d', default='captured')
|
||||
parser.add_argument('--out', '-o', default='parsed_topics.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = {}
|
||||
files = glob(os.path.join(args.dir, '**', '*.json'), recursive=True)
|
||||
logging.info('Scanning %d JSON files under %s', len(files), args.dir)
|
||||
for f in files:
|
||||
items = process_file(f)
|
||||
if items:
|
||||
results[f] = items
|
||||
|
||||
with open(args.out, 'w', encoding='utf-8') as fh:
|
||||
json.dump(results, fh, indent=2)
|
||||
logging.info('Wrote parsed results to %s. %d files had matches.', args.out, len(results))
|
||||
|
||||
# Print a short summary
|
||||
for path, items in results.items():
|
||||
print('\nFile:', path)
|
||||
for it in items[:10]:
|
||||
print(' -', it['key'], ':', it['value'][:120])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Save XHR/fetch requests and responses (including POST bodies).
|
||||
|
||||
Usage:
|
||||
1) Start the script with a persistent profile so you can log in manually:
|
||||
python scrapers/save_xhr_requests.py --profile .playwright_profile --output xhr_captured --domain app.statdx.com
|
||||
2) A browser opens. Log in and navigate to the page that issues XHRs.
|
||||
3) Return to the terminal and press Enter to begin capturing. The script will reload the current page to re-trigger XHRs.
|
||||
4) When you're done, press Ctrl+C; captured request/response pairs will be saved in the output directory.
|
||||
|
||||
Files produced:
|
||||
- <sanitized-url>_<hash>_<timestamp>.json -- metadata (request, response headers, status, any textual body)
|
||||
- <same>.<ext> -- binary response body saved when non-text content (images, etc.)
|
||||
|
||||
The saved JSON includes (where available): url, method, status, request_headers, request_post_data, response_headers, response_text (if textual), response_body_file (if binary saved).
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
import mimetypes
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
|
||||
def save_binary(body: bytes, out_dir: str, url: str, ts: str, ctype: str):
|
||||
ext = ''
|
||||
try:
|
||||
ext = mimetypes.guess_extension(ctype.split(';')[0]) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
fname = f"{sanitize_filename(url)}_{ts}{ext}"
|
||||
path = os.path.join(out_dir, fname)
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile')
|
||||
parser.add_argument('--output', '-o', default='xhr_captured')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--topics-url', default='https://app.statdx.com/')
|
||||
parser.add_argument('--wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium
|
||||
logging.info('Launching persistent context with profile: %s (channel=%s)', args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False, channel=args.channel)
|
||||
else:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False)
|
||||
page = context.new_page()
|
||||
|
||||
captured = []
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = req.resource_type
|
||||
if args.domain not in req.url:
|
||||
return
|
||||
if rtype not in ('xhr', 'fetch'):
|
||||
return
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
info = {
|
||||
'url': req.url,
|
||||
'method': req.method,
|
||||
'resource_type': rtype,
|
||||
'status': resp.status,
|
||||
'request_headers': dict(req.headers),
|
||||
'response_headers': dict(resp.headers),
|
||||
'timestamp': ts,
|
||||
}
|
||||
|
||||
# Request post data (may be None)
|
||||
try:
|
||||
post = req.post_data
|
||||
info['request_post_data'] = post
|
||||
except Exception:
|
||||
try:
|
||||
info['request_post_data'] = req.post_data_text()
|
||||
except Exception:
|
||||
info['request_post_data'] = None
|
||||
|
||||
# Response body
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
try:
|
||||
# prefer text for textual content
|
||||
if 'application/json' in ctype or 'text/' in ctype or 'application/javascript' in ctype:
|
||||
txt = resp.text()
|
||||
info['response_text'] = txt
|
||||
info['response_body_file'] = None
|
||||
else:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
# fallback: save binary
|
||||
try:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
logging.exception('Failed to read response body for %s', req.url)
|
||||
|
||||
fname = f"{sanitize_filename(req.url)}_{ts}.json"
|
||||
out_path = os.path.join(args.output, fname)
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
import json
|
||||
|
||||
json.dump(info, fh, indent=2)
|
||||
|
||||
logging.info('Captured XHR: %s -> %s', req.url, out_path)
|
||||
captured.append(out_path)
|
||||
except Exception:
|
||||
logging.exception('Error in on_response')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
logging.info('Open the browser, log in if needed, and navigate to the page that triggers XHRs. Then return here and press Enter.')
|
||||
page.goto(args.topics_url)
|
||||
input('Press Enter to start capture (will reload the current page)...')
|
||||
|
||||
logging.info('Reloading page to trigger XHRs...')
|
||||
page.reload()
|
||||
|
||||
try:
|
||||
time.sleep(args.wait)
|
||||
logging.info('Waiting finished. You can interact to trigger more XHRs; press Ctrl+C when done.')
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Finishing capture...')
|
||||
finally:
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
logging.info('Done. Captured %d XHRs. Files saved under %s', len(captured), args.output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture-after-login helper
|
||||
|
||||
Usage:
|
||||
1. Run this script. It will open a visible browser with a persistent profile (so you can log in manually).
|
||||
2. Log in to STATdx in the opened browser and navigate to the topics page normally.
|
||||
3. Switch back to the terminal and press Enter. The script will reload the topics URL to trigger API calls,
|
||||
capture XHR/fetch response bodies, save them under the `output` directory, and dump cookies and localStorage.
|
||||
|
||||
Example:
|
||||
python scrapers/statdx_capture_after_login.py --profile .playwright_profile --output captured --domain app.statdx.com --topics-url https://app.statdx.com/
|
||||
|
||||
This is intended to capture the API JSON that the client uses to render pages after authentication.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
|
||||
def dump_storage_and_cookies(context, page, out_dir: str):
|
||||
try:
|
||||
cookies = context.cookies()
|
||||
with open(os.path.join(out_dir, 'cookies.json'), 'w', encoding='utf-8') as fh:
|
||||
json.dump(cookies, fh, indent=2)
|
||||
logging.info('Saved cookies to cookies.json')
|
||||
except Exception:
|
||||
logging.exception('Failed to dump cookies')
|
||||
|
||||
try:
|
||||
local = page.evaluate('()=> JSON.stringify(localStorage)')
|
||||
session = page.evaluate('()=> JSON.stringify(sessionStorage)')
|
||||
with open(os.path.join(out_dir, 'localStorage.json'), 'w', encoding='utf-8') as fh:
|
||||
fh.write(local)
|
||||
with open(os.path.join(out_dir, 'sessionStorage.json'), 'w', encoding='utf-8') as fh:
|
||||
fh.write(session)
|
||||
logging.info('Saved localStorage and sessionStorage')
|
||||
except Exception:
|
||||
logging.exception('Failed to dump storage')
|
||||
|
||||
|
||||
def save_response_body(resp, out_dir: str):
|
||||
try:
|
||||
url = resp.url
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
fname = sanitize_filename(url)
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
if 'application/json' in ctype or url.endswith('.json'):
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.json")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved JSON response: %s', path)
|
||||
return True
|
||||
elif 'text/html' in ctype or resp.request.resource_type == 'document':
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.html")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved HTML response: %s', path)
|
||||
return True
|
||||
elif ctype.startswith('text/'):
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.txt")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved text response: %s', path)
|
||||
return True
|
||||
else:
|
||||
# Save binary content (images, fonts, etc.)
|
||||
try:
|
||||
body = resp.body()
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}")
|
||||
# try to add extension from content-type
|
||||
ctype_main = resp.headers.get('content-type', '').split(';')[0]
|
||||
try:
|
||||
import mimetypes
|
||||
|
||||
ext = mimetypes.guess_extension(ctype_main) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
if ext:
|
||||
path += ext
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
logging.info('Saved binary response: %s', path)
|
||||
return True
|
||||
except Exception:
|
||||
logging.exception('Failed to save binary response')
|
||||
return False
|
||||
except Exception:
|
||||
logging.exception('Failed to save response body')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile')
|
||||
parser.add_argument('--output', '-o', default='captured')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--topics-url', default='https://app.statdx.com/')
|
||||
parser.add_argument('--wait', type=float, default=8.0, help='Seconds to wait after reload to capture responses')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium
|
||||
logging.info('Launching persistent context with profile: %s (channel=%s)', args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False, channel=args.channel)
|
||||
else:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False)
|
||||
page = context.new_page()
|
||||
|
||||
captured = []
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
if args.domain not in resp.url:
|
||||
return
|
||||
rtype = resp.request.resource_type
|
||||
# focus on XHR/fetch and document
|
||||
if rtype in ('xhr', 'fetch', 'document'):
|
||||
ok = save_response_body(resp, args.output)
|
||||
if ok:
|
||||
captured.append(resp.url)
|
||||
except Exception:
|
||||
logging.exception('response handler error')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
logging.info('Open browser and log in if needed. Navigate to the topics page. Then return to the terminal and press Enter to trigger capture.')
|
||||
page.goto(args.topics_url)
|
||||
|
||||
input('Press Enter when you are logged in and on the page you want to capture...')
|
||||
|
||||
# Reload to trigger API calls while our handler is active
|
||||
logging.info('Reloading topics URL to trigger API calls...')
|
||||
page.reload()
|
||||
|
||||
# Optionally auto-scroll to trigger lazy loads
|
||||
try:
|
||||
page.evaluate(
|
||||
"""
|
||||
async () => {
|
||||
const wait = ms => new Promise(r => setTimeout(r, ms));
|
||||
const doc = document.scrollingElement || document.documentElement;
|
||||
let total = 0;
|
||||
while (total < doc.scrollHeight) {
|
||||
doc.scrollBy(0, 400);
|
||||
total += 400;
|
||||
await wait(150);
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
logging.exception('auto-scroll failed during capture')
|
||||
|
||||
logging.info('Waiting %.1f seconds for responses...', args.wait)
|
||||
time.sleep(args.wait)
|
||||
|
||||
dump_storage_and_cookies(context, page, args.output)
|
||||
|
||||
# Save a final snapshot of the rendered page
|
||||
try:
|
||||
html = page.content()
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
fname = sanitize_filename(page.url)
|
||||
snap_path = os.path.join(args.output, f'snapshot_after_login_{fname}_{ts}.html')
|
||||
with open(snap_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(html)
|
||||
logging.info('Saved final page snapshot: %s', snap_path)
|
||||
# screenshots disabled per user request
|
||||
except Exception:
|
||||
logging.exception('Failed to save final snapshot')
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
logging.info('Done. Captured %d responses.', len(captured))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnostic capture script: navigates to the site headful, records console/page errors, and logs requests and DNS checks.
|
||||
|
||||
Run:
|
||||
python scrapers/statdx_debug_capture.py --domain app.statdx.com --output debug_report.txt
|
||||
|
||||
It will open a visible browser; interact briefly, then press Ctrl+C in the terminal to finish and write the report.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime
|
||||
from datetime import timezone
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def dns_check(host):
|
||||
try:
|
||||
info = socket.getaddrinfo(host, None)
|
||||
addrs = sorted({i[4][0] for i in info})
|
||||
return True, addrs
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--output', '-o', default='debug_report.txt')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
report = []
|
||||
report.append(f'Report generated: {datetime.now(timezone.utc).isoformat()}Z')
|
||||
report.append(f'Domain: {args.domain}')
|
||||
|
||||
# DNS check for the main domain and some common vendors
|
||||
hosts = [args.domain, 'app.pendo.io', 'static.cloudflareinsights.com']
|
||||
report.append('\nDNS checks:')
|
||||
for h in hosts:
|
||||
ok, res = dns_check(h)
|
||||
report.append(f' - {h}: {ok} -> {res}')
|
||||
|
||||
requests = []
|
||||
console_logs = []
|
||||
page_errors = []
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=args.headless)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
def on_request(req):
|
||||
requests.append({'url': req.url, 'method': req.method, 'resource_type': req.resource_type})
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
requests.append({'url': resp.url, 'status': resp.status, 'resource_type': resp.request.resource_type})
|
||||
except Exception:
|
||||
requests.append({'url': getattr(resp, 'url', '<unknown>'), 'status': 'error'})
|
||||
|
||||
def on_console(msg):
|
||||
console_logs.append({'type': msg.type, 'text': msg.text})
|
||||
|
||||
def on_page_error(err):
|
||||
page_errors.append(str(err))
|
||||
|
||||
page.on('request', on_request)
|
||||
page.on('response', on_response)
|
||||
page.on('console', on_console)
|
||||
page.on('pageerror', on_page_error)
|
||||
|
||||
logging.info('Opening https://%s/', args.domain)
|
||||
page.goto(f'https://{args.domain}/')
|
||||
logging.info('Interact with the page now. Press Ctrl+C in the terminal to finish and write the report.')
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Finishing and writing report...')
|
||||
finally:
|
||||
try:
|
||||
browser.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing browser (driver may have disconnected) — continuing to write report')
|
||||
|
||||
report.append('\nConsole logs:')
|
||||
for c in console_logs:
|
||||
report.append(f" - [{c['type']}] {c['text']}")
|
||||
|
||||
report.append('\nPage errors:')
|
||||
for e in page_errors:
|
||||
report.append(' - ' + e)
|
||||
|
||||
report.append('\nRecent requests (last 100):')
|
||||
for r in requests[-100:]:
|
||||
if isinstance(r, dict):
|
||||
report.append(' - {status} {resource_type} {url}'.format(status=r.get('status', ''), resource_type=r.get('resource_type', ''), url=r.get('url')))
|
||||
else:
|
||||
report.append(' - ' + str(r))
|
||||
|
||||
with open(args.output, 'w', encoding='utf-8') as fh:
|
||||
fh.write('\n'.join(report))
|
||||
|
||||
print('Wrote report to', args.output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live-capture browser: open a visible Chromium window, persist the profile, and save HTML/JSON responses
|
||||
as you browse so you can "download content as you browse normally".
|
||||
|
||||
Usage:
|
||||
- Install Playwright and browsers: python -m pip install playwright python-dotenv; python -m playwright install
|
||||
- Run:
|
||||
python scrapers/statdx_live_capture.py --output captured --domain app.statdx.com
|
||||
|
||||
What it does:
|
||||
- Launches a persistent Chromium context (profile saved to .playwright_profile by default) so cookies/session persist.
|
||||
- Opens a visible browser (headful). You can interact with it manually.
|
||||
- Listens to network responses. When it sees a response whose content-type is HTML or JSON (or document resource),
|
||||
it saves the response body to the `output` directory with a sanitized filename derived from the URL.
|
||||
|
||||
Notes / limitations:
|
||||
- This saves HTTP responses as they arrive. It does not run playback for dynamic client-rendered content (but it
|
||||
will save the HTML shell and any XHR/JSON responses used by the client).
|
||||
- You may need to filter by domain to avoid saving lots of third-party assets.
|
||||
- Respect terms of service and only use for accounts you own/have permission to access.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import threading
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
from dotenv import load_dotenv
|
||||
import mimetypes
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
"""Create a short filesystem-safe filename from a URL."""
|
||||
parsed = urlparse(url)
|
||||
# use host + path + query, but keep it short
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += "?" + parsed.query
|
||||
# remove leading/trailing slashes
|
||||
base = base.strip("/")
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
# replace problematic chars
|
||||
safe = "".join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
# add a short hash to avoid collisions and trim length
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
filename = f"{safe[:240]}_{h}"
|
||||
return filename
|
||||
|
||||
|
||||
def auto_scroll(page, step_ms: int = 50, step_px: int = 300):
|
||||
"""Scroll the page to the bottom slowly to trigger lazy loading.
|
||||
|
||||
Note: executed in the sync Playwright context by calling page.evaluate.
|
||||
"""
|
||||
try:
|
||||
page.evaluate(
|
||||
"""
|
||||
async (step_px, step_ms) => {
|
||||
const wait = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const doc = document.scrollingElement || document.documentElement;
|
||||
let total = 0;
|
||||
while (total < doc.scrollHeight) {
|
||||
doc.scrollBy(0, step_px);
|
||||
total += step_px;
|
||||
await wait(step_ms);
|
||||
}
|
||||
}
|
||||
""",
|
||||
step_px,
|
||||
step_ms,
|
||||
)
|
||||
except Exception:
|
||||
logging.exception('auto_scroll failed')
|
||||
|
||||
|
||||
def save_response(resp, out_dir: str):
|
||||
try:
|
||||
url = resp.url
|
||||
headers = resp.headers
|
||||
ctype = headers.get('content-type', '')
|
||||
|
||||
# Decide whether to save: HTML or JSON (or document)
|
||||
save_as_text = False
|
||||
save_as_binary = False
|
||||
|
||||
if 'text/html' in ctype or resp.request.resource_type == 'document':
|
||||
save_as_text = True
|
||||
ext = '.html'
|
||||
elif 'application/json' in ctype or 'text/javascript' in ctype or 'application/javascript' in ctype:
|
||||
save_as_text = True
|
||||
ext = '.json'
|
||||
elif ctype.startswith('text/'):
|
||||
save_as_text = True
|
||||
ext = '.txt'
|
||||
else:
|
||||
# treat as binary and save (images, fonts, video, etc.)
|
||||
save_as_binary = True
|
||||
ext = ''
|
||||
|
||||
if not (save_as_text or save_as_binary):
|
||||
return False
|
||||
|
||||
filename_base = sanitize_filename(url)
|
||||
out_path = os.path.join(out_dir, filename_base + ext)
|
||||
|
||||
logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0])
|
||||
|
||||
if save_as_text:
|
||||
try:
|
||||
body = resp.text()
|
||||
except Exception:
|
||||
# fallback to binary then decode
|
||||
body = resp.body().decode('utf-8', errors='replace')
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(body)
|
||||
else:
|
||||
body = resp.body()
|
||||
# try to guess extension from content-type
|
||||
try:
|
||||
ctype_main = ctype.split(';')[0]
|
||||
ext_guess = mimetypes.guess_extension(ctype_main) or ''
|
||||
except Exception:
|
||||
ext_guess = ''
|
||||
out_path = out_path + (ext_guess if ext_guess else '')
|
||||
with open(out_path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.exception('Failed to save response: %s', e)
|
||||
return False
|
||||
|
||||
|
||||
def save_page_snapshot(page, out_dir: str, reason: str = ''):
|
||||
try:
|
||||
url = page.url
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
filename_base = sanitize_filename(url)
|
||||
safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40]
|
||||
out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html")
|
||||
logging.info('Saving page snapshot: %s -> %s', url, out_path)
|
||||
content = page.content()
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(content)
|
||||
# Screenshots are disabled (not saving visual snapshots per user request)
|
||||
return True
|
||||
except Exception:
|
||||
logging.exception('Failed to save page snapshot')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright')
|
||||
parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses')
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory')
|
||||
parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain')
|
||||
parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)')
|
||||
parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot')
|
||||
parser.add_argument('--devtools', action='store_true', help='Open browser with devtools panel')
|
||||
parser.add_argument('--width', type=int, default=1280, help='Viewport width')
|
||||
parser.add_argument('--height', type=int, default=1600, help='Viewport height')
|
||||
parser.add_argument('--slowmo', type=int, default=0, help='Slow down Playwright actions (ms)')
|
||||
parser.add_argument('--user-agent', dest='user_agent', help='Custom User-Agent header')
|
||||
parser.add_argument('--wait-for-selector', dest='wait_for_selector', help='CSS selector to wait for after navigation')
|
||||
parser.add_argument('--auto-scroll', action='store_true', help='Automatically scroll the page when snapshotting to trigger lazy loads')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
# Use persistent context so cookies/localStorage persist between runs
|
||||
browser_type = p.chromium
|
||||
logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
|
||||
else:
|
||||
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
# Optionally you can set viewport, user agent, etc. here.
|
||||
# Create a new page with the requested viewport
|
||||
page = context.new_page()
|
||||
try:
|
||||
page.set_viewport_size({"width": args.width, "height": args.height})
|
||||
except Exception:
|
||||
# Older playwright versions may not support set_viewport_size on persistent contexts
|
||||
pass
|
||||
|
||||
def handle_response(resp):
|
||||
try:
|
||||
if args.domain not in resp.url:
|
||||
return
|
||||
# save textual responses
|
||||
saved = save_response(resp, args.output)
|
||||
# If this response is a document or an XHR/fetch that likely carried data,
|
||||
# also save a rendered page snapshot so client-side content is captured.
|
||||
rtype = resp.request.resource_type
|
||||
if saved and rtype in ("document", "xhr", "fetch"):
|
||||
# page may not have finished rendering; give it a small chance to update
|
||||
try:
|
||||
time.sleep(args.snapshot_delay)
|
||||
if args.auto_scroll:
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason=rtype)
|
||||
except Exception:
|
||||
logging.exception('snapshot after response failed')
|
||||
except Exception:
|
||||
logging.exception('Error in response handler')
|
||||
|
||||
|
||||
page.on('response', handle_response)
|
||||
|
||||
# Log console messages and page errors to help diagnose missing content
|
||||
def on_console(msg):
|
||||
try:
|
||||
logging.info('PAGE LOG [%s] %s', msg.type, msg.text)
|
||||
except Exception:
|
||||
logging.info('PAGE LOG: %s', msg)
|
||||
|
||||
def on_page_error(exc):
|
||||
logging.error('PAGE ERROR: %s', exc)
|
||||
|
||||
page.on('console', on_console)
|
||||
page.on('pageerror', on_page_error)
|
||||
|
||||
logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output)
|
||||
logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
|
||||
logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
|
||||
|
||||
# Navigate to the domain root so you can log in manually if needed
|
||||
start_url = f'https://{args.domain}/'
|
||||
page.goto(start_url)
|
||||
# Optionally wait for a selector that indicates the page finished rendering
|
||||
if args.wait_for_selector:
|
||||
try:
|
||||
logging.info('Waiting for selector: %s', args.wait_for_selector)
|
||||
page.wait_for_selector(args.wait_for_selector, timeout=20000)
|
||||
except Exception:
|
||||
logging.exception('Waiting for selector timed out')
|
||||
|
||||
def stdin_watcher():
|
||||
# Run in a background thread so the main thread can keep the browser running
|
||||
logging.debug('stdin watcher started')
|
||||
for line in sys.stdin:
|
||||
cmd = line.strip().lower()
|
||||
if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save':
|
||||
logging.info('Manual snapshot requested via stdin')
|
||||
try:
|
||||
if args.auto_scroll:
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('manual auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason='manual')
|
||||
except Exception:
|
||||
logging.exception('manual snapshot failed')
|
||||
|
||||
watcher = threading.Thread(target=stdin_watcher, daemon=True)
|
||||
watcher.start()
|
||||
|
||||
try:
|
||||
# Keep the script running while the user interacts with the headed browser
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Shutting down...')
|
||||
finally:
|
||||
context.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Launch the site in an actual installed Chrome via Playwright's 'chrome' channel.
|
||||
|
||||
This can help if the headful Playwright Chromium environment differs from your regular Chrome and the site
|
||||
only fully renders in a real browser environment.
|
||||
|
||||
Usage:
|
||||
python scrapers/statdx_play_chrome.py --profile .chrome_profile --url https://app.statdx.com/ --screenshot out.png
|
||||
|
||||
Notes:
|
||||
- Requires Playwright and that the 'chrome' browser is available on your machine.
|
||||
- If Playwright cannot find the channel, install Chrome or use the system browser path.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
from time import sleep
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from datetime import datetime, timezone
|
||||
from playwright.sync_api import sync_playwright
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env in the repository root (if present)
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.chrome_profile')
|
||||
parser.add_argument('--url', '-u', default='https://app.statdx.com/')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com', help='Domain to filter captures')
|
||||
parser.add_argument('--screenshot', '-s', default='chrome_page.png')
|
||||
parser.add_argument('--wait-for', help='Optional selector to wait for before screenshot')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
|
||||
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
|
||||
parser.add_argument('--post-login-selector', default='#ds-app', help='Selector indicating successful login (default: #ds-app)')
|
||||
parser.add_argument('--wait-after-login', type=float, default=3.0, help='Seconds to wait after login before snapshot')
|
||||
parser.add_argument('--keep-open', action='store_true', help='Leave the browser open after actions')
|
||||
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
|
||||
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
|
||||
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.profile, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
try:
|
||||
logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
|
||||
except Exception as e:
|
||||
logging.error('Failed to launch Chrome channel: %s', e)
|
||||
logging.info('Falling back to default Chromium persistent context')
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
page = context.new_page()
|
||||
page.goto(args.url)
|
||||
|
||||
# Setup capture if requested
|
||||
captured = []
|
||||
if args.capture:
|
||||
os.makedirs(args.capture_output, exist_ok=True)
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
def save_binary(body: bytes, out_dir: str, url: str, ts: str, ctype: str):
|
||||
ext = ''
|
||||
try:
|
||||
ext = mimetypes.guess_extension(ctype.split(';')[0]) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
fname = f"{sanitize_filename(url)}_{ts}{ext}"
|
||||
path = os.path.join(out_dir, fname)
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
return path
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = req.resource_type
|
||||
if args.domain not in req.url:
|
||||
return
|
||||
if rtype not in ('xhr', 'fetch'):
|
||||
return
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
info = {
|
||||
'url': req.url,
|
||||
'method': req.method,
|
||||
'resource_type': rtype,
|
||||
'status': resp.status,
|
||||
'request_headers': dict(req.headers),
|
||||
'response_headers': dict(resp.headers),
|
||||
'timestamp': ts,
|
||||
}
|
||||
|
||||
try:
|
||||
post = req.post_data
|
||||
info['request_post_data'] = post
|
||||
except Exception:
|
||||
try:
|
||||
info['request_post_data'] = req.post_data_text()
|
||||
except Exception:
|
||||
info['request_post_data'] = None
|
||||
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
try:
|
||||
if 'application/json' in ctype or 'text/' in ctype or 'application/javascript' in ctype:
|
||||
txt = resp.text()
|
||||
info['response_text'] = txt
|
||||
info['response_body_file'] = None
|
||||
else:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
try:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
logging.exception('Failed to read response body for %s', req.url)
|
||||
|
||||
fname = f"{sanitize_filename(req.url)}_{ts}.json"
|
||||
out_path = os.path.join(args.capture_output, fname)
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
import json
|
||||
|
||||
json.dump(info, fh, indent=2)
|
||||
|
||||
logging.info('Captured XHR: %s -> %s', req.url, out_path)
|
||||
captured.append(out_path)
|
||||
except Exception:
|
||||
logging.exception('Error in capture on_response')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
# Automatic login if credentials provided
|
||||
username = args.username or os.getenv('STATDX_USERNAME')
|
||||
password = args.password or os.getenv('STATDX_PASSWORD')
|
||||
if username and password:
|
||||
logging.info('Attempting automatic login using provided credentials')
|
||||
# candidate selectors for username and password fields
|
||||
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
|
||||
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
||||
|
||||
user_sel = None
|
||||
for sel in user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
pass_sel = None
|
||||
for sel in pass_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
pass_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel or not pass_sel:
|
||||
logging.warning('Could not auto-detect username/password fields. Skipping automatic login.')
|
||||
else:
|
||||
try:
|
||||
page.fill(user_sel, username)
|
||||
sleep(0.1)
|
||||
page.fill(pass_sel, password)
|
||||
sleep(0.1)
|
||||
# try submit buttons
|
||||
submit_selectors = ['button.primary.submitSelector', 'button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
|
||||
clicked = False
|
||||
for s in submit_selectors:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
logging.info('Clicked submit button using selector: %s', s)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if not clicked:
|
||||
logging.info('No submit button clicked; pressing Enter in password field')
|
||||
page.press(pass_sel, 'Enter')
|
||||
|
||||
# wait for either navigation or a post-login selector
|
||||
try:
|
||||
if args.post_login_selector:
|
||||
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
|
||||
except Exception:
|
||||
logging.info('Post-login selector not found within timeout')
|
||||
|
||||
# small extra wait to let JS render
|
||||
sleep(args.wait_after_login)
|
||||
except Exception:
|
||||
logging.exception('Automatic login attempt failed')
|
||||
|
||||
if args.wait_for:
|
||||
try:
|
||||
logging.info('Waiting for selector: %s', args.wait_for)
|
||||
page.wait_for_selector(args.wait_for, timeout=20000)
|
||||
except Exception:
|
||||
logging.exception('wait_for selector timed out')
|
||||
|
||||
# give some time for JS to finish
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
# Screenshots are disabled per user request
|
||||
|
||||
if args.keep_open:
|
||||
logging.info('Leaving browser open for manual inspection. Close the browser window to finish.')
|
||||
try:
|
||||
# keep running until user closes browser
|
||||
context.wait_for_event('close')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Playwright-based login + scraping for STATdx (recommended for JS-heavy sites).
|
||||
|
||||
This script launches a headless Chromium, navigates to the app, fills username/password, clicks the submit button,
|
||||
waits for a post-login selector, then saves the resulting HTML to disk.
|
||||
|
||||
You may need to tweak selectors if the login form uses unusual markup or an SSO redirect.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
USERNAME = os.getenv("STATDX_USERNAME")
|
||||
PASSWORD = os.getenv("STATDX_PASSWORD")
|
||||
TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
||||
|
||||
if not USERNAME or not PASSWORD:
|
||||
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
|
||||
def run(headless=True, timeout=30000):
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=headless)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
logging.info("Opening %s", TARGET_URL)
|
||||
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
|
||||
|
||||
# Heuristics to find fields — adapt per the real login form
|
||||
password_selector = "input[type=\"password\"]"
|
||||
try:
|
||||
page.wait_for_selector(password_selector, timeout=8000)
|
||||
logging.info("Password field found")
|
||||
except Exception:
|
||||
logging.warning("Password field not found quickly; continuing to try common selectors")
|
||||
|
||||
# Try to detect username/email field by common selectors
|
||||
candidate_user_selectors = [
|
||||
"input[type=\"email\"]",
|
||||
"input[name*=\"email\" i]",
|
||||
"input[name*=\"user\" i]",
|
||||
"input[type=\"text\"]",
|
||||
]
|
||||
|
||||
user_sel = None
|
||||
for sel in candidate_user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel:
|
||||
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
|
||||
# fallback: pick first visible text input
|
||||
inputs = page.query_selector_all("input[type=\"text\"]")
|
||||
if inputs:
|
||||
user_sel = "input[type=\"text\"]"
|
||||
|
||||
logging.info("Using username selector: %s", user_sel)
|
||||
|
||||
if user_sel:
|
||||
page.fill(user_sel, USERNAME)
|
||||
else:
|
||||
logging.error("No username field found. Aborting.")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
# Fill password
|
||||
if page.query_selector(password_selector):
|
||||
page.fill(password_selector, PASSWORD)
|
||||
else:
|
||||
logging.error("No password field found. Aborting.")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
# Try to click a submit button
|
||||
submit_selectors = [
|
||||
"button[type=submit]",
|
||||
"input[type=submit]",
|
||||
"button:has-text(\"Sign in\")",
|
||||
"button:has-text(\"Sign In\")",
|
||||
"button:has-text(\"Log in\")",
|
||||
"button:has-text(\"Log In\")",
|
||||
]
|
||||
|
||||
clicked = False
|
||||
for s in submit_selectors:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
logging.info("Clicked submit via selector %s", s)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not clicked:
|
||||
logging.warning("No submit button clicked; attempting to press Enter on password field")
|
||||
page.press(password_selector, "Enter")
|
||||
|
||||
# Wait for navigation or the app root element that indicates successful login
|
||||
try:
|
||||
page.wait_for_selector("#ds-app", timeout=20000)
|
||||
logging.info("#ds-app found — likely logged in and app rendered")
|
||||
except Exception:
|
||||
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
|
||||
|
||||
# Optional: wait a bit for client-side rendering
|
||||
time.sleep(2)
|
||||
|
||||
content = page.content()
|
||||
out_file = "topics_playwright.html"
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
logging.info("Saved rendered HTML to %s", out_file)
|
||||
|
||||
browser.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(headless=True)
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Form-based login + scraping example for STATdx (best-effort).
|
||||
|
||||
This script attempts to:
|
||||
- GET the login page
|
||||
- detect a login form and its fields (username/password)
|
||||
- submit credentials and preserve the session cookie
|
||||
- fetch the target topics page and print/save HTML
|
||||
|
||||
Notes:
|
||||
- Most modern sites use JS-driven auth or SSO; if this fails, use the Playwright script.
|
||||
- Inspect the login POST URL and field names with your browser and set the environment overrides in .env
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
# Configuration via environment (or fallbacks)
|
||||
LOGIN_PAGE = os.getenv("STATDX_LOGIN_URL", "https://app.statdx.com/")
|
||||
TOPICS_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
||||
USERNAME = os.getenv("STATDX_USERNAME")
|
||||
PASSWORD = os.getenv("STATDX_PASSWORD")
|
||||
OVERRIDE_USER_FIELD = os.getenv("STATDX_USERNAME_FIELD")
|
||||
OVERRIDE_PASS_FIELD = os.getenv("STATDX_PASSWORD_FIELD")
|
||||
|
||||
if not USERNAME or not PASSWORD:
|
||||
logging.error("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
||||
sys.exit(1)
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
def find_login_form(html, base_url):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
forms = soup.find_all("form")
|
||||
if not forms:
|
||||
return None
|
||||
|
||||
# Heuristic: pick the form that contains an input of type password
|
||||
for form in forms:
|
||||
if form.find("input", {"type": "password"}):
|
||||
action = form.get("action") or base_url
|
||||
method = (form.get("method") or "get").lower()
|
||||
inputs = {}
|
||||
for inp in form.find_all("input"):
|
||||
name = inp.get("name")
|
||||
if not name:
|
||||
continue
|
||||
value = inp.get("value", "")
|
||||
inputs[name] = value
|
||||
return {"action": urljoin(base_url, action), "method": method, "inputs": inputs}
|
||||
return None
|
||||
|
||||
|
||||
def guess_field_names(inputs):
|
||||
# Try to find which input names likely correspond to username and password
|
||||
u_name = None
|
||||
p_name = None
|
||||
for name in inputs:
|
||||
ln = name.lower()
|
||||
if any(x in ln for x in ("user", "email", "login")) and not u_name:
|
||||
u_name = name
|
||||
if "pass" in ln and not p_name:
|
||||
p_name = name
|
||||
# Fallback to common names
|
||||
if not u_name:
|
||||
for cand in ("username", "email", "user"):
|
||||
if cand in inputs:
|
||||
u_name = cand
|
||||
break
|
||||
if not p_name:
|
||||
for cand in ("password", "pass"):
|
||||
if cand in inputs:
|
||||
p_name = cand
|
||||
break
|
||||
return u_name, p_name
|
||||
|
||||
|
||||
def login_using_form(login_page_url):
|
||||
logging.info("Fetching login page: %s", login_page_url)
|
||||
r = session.get(login_page_url, allow_redirects=True)
|
||||
r.raise_for_status()
|
||||
form = find_login_form(r.text, r.url)
|
||||
if not form:
|
||||
logging.warning("No login form found on page. The site might be JS-driven. Consider the Playwright script.")
|
||||
return False
|
||||
|
||||
action = form["action"]
|
||||
method = form["method"]
|
||||
inputs = form["inputs"].copy()
|
||||
|
||||
# Determine username / password field names
|
||||
user_field = OVERRIDE_USER_FIELD or guess_field_names(inputs)[0]
|
||||
pass_field = OVERRIDE_PASS_FIELD or guess_field_names(inputs)[1]
|
||||
|
||||
if not user_field or not pass_field:
|
||||
logging.error("Could not detect username/password field names. Please inspect the login form and set STATDX_USERNAME_FIELD and STATDX_PASSWORD_FIELD in .env")
|
||||
return False
|
||||
|
||||
inputs[user_field] = USERNAME
|
||||
inputs[pass_field] = PASSWORD
|
||||
|
||||
logging.info("Submitting login form to: %s", action)
|
||||
if method == "post":
|
||||
resp = session.post(action, data=inputs, allow_redirects=True)
|
||||
else:
|
||||
resp = session.get(action, params=inputs, allow_redirects=True)
|
||||
|
||||
logging.info("Login response: %s %s", resp.status_code, resp.url)
|
||||
|
||||
# Basic success check: did we land on a different URL or see a known element?
|
||||
if resp.url != login_page_url and resp.status_code in (200, 302, 303):
|
||||
logging.info("Looks like login may have succeeded (redirected).")
|
||||
return True
|
||||
|
||||
# Heuristic: search for 'logout' or 'sign out' in HTML
|
||||
if "logout" in resp.text.lower() or "sign out" in resp.text.lower():
|
||||
logging.info("Found logout text — login probably succeeded.")
|
||||
return True
|
||||
|
||||
logging.warning("Login may have failed (no redirect or logout text). Try Playwright or provide exact POST URL/fields.)")
|
||||
return False
|
||||
|
||||
|
||||
def fetch_topics(url):
|
||||
logging.info("Fetching topics page: %s", url)
|
||||
r = session.get(url)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
||||
def save_html(content, path="topics.html"):
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
logging.info("Saved HTML to %s", path)
|
||||
|
||||
|
||||
def main():
|
||||
ok = login_using_form(LOGIN_PAGE)
|
||||
if not ok:
|
||||
logging.error("Form-based login failed. If the site uses JS/SSO, try the Playwright script: scrapers/statdx_playwright.py")
|
||||
return
|
||||
|
||||
html = fetch_topics(TOPICS_URL)
|
||||
save_html(html)
|
||||
# As a minimal parser example, show the contents of <div id="ds-app">
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
ds_app = soup.find(id="ds-app")
|
||||
if ds_app:
|
||||
logging.info("Found #ds-app — page likely relies on JavaScript. For rendered content use Playwright to get client-side-rendered HTML.")
|
||||
else:
|
||||
logging.info("#ds-app not found (or empty). You can parse the HTML above for static content.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user