Add Playwright-based scraping and login scripts for STATdx
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
This commit is contained in:
Ross
+70
@@ -0,0 +1,70 @@
|
||||
# Python cache and bytecode
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
|
||||
# Virtual environments
|
||||
env/
|
||||
venv/
|
||||
.venv/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Distribution / packaging
|
||||
build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
.eggs/
|
||||
pip-wheel-metadata/
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Test, coverage, caches
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache/
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
.hypothesis/
|
||||
|
||||
# IDEs and editors
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Logs and databases
|
||||
*.log
|
||||
*.sqlite3
|
||||
*.db
|
||||
|
||||
# Environment/secrets
|
||||
.env
|
||||
.env.*
|
||||
secrets.json
|
||||
|
||||
# Compiled extensions
|
||||
*.so
|
||||
*.dylib
|
||||
*.dll
|
||||
|
||||
# Jupyter
|
||||
.ipynb_checkpoints/
|
||||
|
||||
# Misc
|
||||
*.egg
|
||||
*.egg-info
|
||||
*.dist-info
|
||||
|
||||
.chrome_profile
|
||||
.playwright_profile
|
||||
|
||||
captured/
|
||||
@@ -0,0 +1,54 @@
|
||||
STATdx scraping helpers
|
||||
|
||||
This folder contains two example Python scripts showing common ways to programmatically log in and scrape pages from a site that requires authentication:
|
||||
|
||||
- `scrapers/statdx_requests.py` — a "requests"-based approach that works when authentication is a plain form POST (non-JS or CSRF-protected with a token present in the HTML). It tries to discover the login form and submit it. You will likely need to update the login action URL or the form field names for STATdx.
|
||||
|
||||
- `scrapers/statdx_playwright.py` — a Playwright-based, browser-automation approach which runs a real Chromium instance, so it works with JS-heavy sites and single-sign-on flows.
|
||||
|
||||
Files:
|
||||
- `requirements.txt` — Python deps
|
||||
- `.env.example` — environment variable names to copy into `.env`
|
||||
- `scrapers/statdx_requests.py` — example script
|
||||
- `scrapers/statdx_playwright.py` — playwright script
|
||||
|
||||
Security / Legal
|
||||
- Only run these scripts for accounts/data you own or have explicit permission to access.
|
||||
- Do not hard-code credentials into scripts. Use environment variables or a secrets manager.
|
||||
|
||||
Quick start (fish shell):
|
||||
|
||||
1) Create a virtualenv and install deps:
|
||||
|
||||
```fish
|
||||
python -m venv .venv
|
||||
. .venv/bin/activate.fish
|
||||
pip install -r requirements.txt
|
||||
# When using Playwright, install browsers once:
|
||||
python -m playwright install
|
||||
```
|
||||
|
||||
2) Copy `.env.example` to `.env` and edit with your username/password:
|
||||
|
||||
```fish
|
||||
cp .env.example .env
|
||||
# edit .env with your editor
|
||||
```
|
||||
|
||||
3) Run the requests script (may need field names):
|
||||
|
||||
```fish
|
||||
python scrapers/statdx_requests.py
|
||||
```
|
||||
|
||||
4) Run the Playwright script (works for JS logins):
|
||||
|
||||
```fish
|
||||
python scrapers/statdx_playwright.py
|
||||
```
|
||||
|
||||
If the login flow fails, open your browser devtools (Network tab) while logging in manually to find the exact POST URL and field names and paste them into the script configuration.
|
||||
|
||||
If you want, I can adapt the scripts further after you either:
|
||||
- share the network POST URL + field names, or
|
||||
- allow me to run the script here with credentials (not possible without your consent)."
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 60 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 336 KiB |
@@ -0,0 +1 @@
|
||||
{}
|
||||
@@ -0,0 +1,4 @@
|
||||
requests>=2.28.0
|
||||
beautifulsoup4>=4.12.0
|
||||
playwright>=1.40.0
|
||||
python-dotenv>=1.0.0
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan captured JSON files and extract likely title/name fields.
|
||||
|
||||
Usage:
|
||||
python scrapers/parse_captured_json.py --dir captured --out parsed_topics.json
|
||||
|
||||
This prints a brief summary to stdout and writes a JSON file with discovered records.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
from glob import glob
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
LIKELY_KEYS = ['title', 'name', 'label', 'displayName', 'heading']
|
||||
|
||||
|
||||
def extract_from_obj(obj):
|
||||
found = []
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in LIKELY_KEYS and isinstance(v, (str, int, float)):
|
||||
found.append({'key': k, 'value': str(v)})
|
||||
else:
|
||||
found.extend(extract_from_obj(v))
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
found.extend(extract_from_obj(item))
|
||||
return found
|
||||
|
||||
|
||||
def process_file(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as fh:
|
||||
data = json.load(fh)
|
||||
except Exception:
|
||||
# skip files that aren't valid JSON
|
||||
return []
|
||||
return extract_from_obj(data)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--dir', '-d', default='captured')
|
||||
parser.add_argument('--out', '-o', default='parsed_topics.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = {}
|
||||
files = glob(os.path.join(args.dir, '**', '*.json'), recursive=True)
|
||||
logging.info('Scanning %d JSON files under %s', len(files), args.dir)
|
||||
for f in files:
|
||||
items = process_file(f)
|
||||
if items:
|
||||
results[f] = items
|
||||
|
||||
with open(args.out, 'w', encoding='utf-8') as fh:
|
||||
json.dump(results, fh, indent=2)
|
||||
logging.info('Wrote parsed results to %s. %d files had matches.', args.out, len(results))
|
||||
|
||||
# Print a short summary
|
||||
for path, items in results.items():
|
||||
print('\nFile:', path)
|
||||
for it in items[:10]:
|
||||
print(' -', it['key'], ':', it['value'][:120])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Save XHR/fetch requests and responses (including POST bodies).
|
||||
|
||||
Usage:
|
||||
1) Start the script with a persistent profile so you can log in manually:
|
||||
python scrapers/save_xhr_requests.py --profile .playwright_profile --output xhr_captured --domain app.statdx.com
|
||||
2) A browser opens. Log in and navigate to the page that issues XHRs.
|
||||
3) Return to the terminal and press Enter to begin capturing. The script will reload the current page to re-trigger XHRs.
|
||||
4) When you're done, press Ctrl+C; captured request/response pairs will be saved in the output directory.
|
||||
|
||||
Files produced:
|
||||
- <sanitized-url>_<hash>_<timestamp>.json -- metadata (request, response headers, status, any textual body)
|
||||
- <same>.<ext> -- binary response body saved when non-text content (images, etc.)
|
||||
|
||||
The saved JSON includes (where available): url, method, status, request_headers, request_post_data, response_headers, response_text (if textual), response_body_file (if binary saved).
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
import mimetypes
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
|
||||
def save_binary(body: bytes, out_dir: str, url: str, ts: str, ctype: str):
|
||||
ext = ''
|
||||
try:
|
||||
ext = mimetypes.guess_extension(ctype.split(';')[0]) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
fname = f"{sanitize_filename(url)}_{ts}{ext}"
|
||||
path = os.path.join(out_dir, fname)
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
return path
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile')
|
||||
parser.add_argument('--output', '-o', default='xhr_captured')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--topics-url', default='https://app.statdx.com/')
|
||||
parser.add_argument('--wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium
|
||||
logging.info('Launching persistent context with profile: %s (channel=%s)', args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False, channel=args.channel)
|
||||
else:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False)
|
||||
page = context.new_page()
|
||||
|
||||
captured = []
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = req.resource_type
|
||||
if args.domain not in req.url:
|
||||
return
|
||||
if rtype not in ('xhr', 'fetch'):
|
||||
return
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
info = {
|
||||
'url': req.url,
|
||||
'method': req.method,
|
||||
'resource_type': rtype,
|
||||
'status': resp.status,
|
||||
'request_headers': dict(req.headers),
|
||||
'response_headers': dict(resp.headers),
|
||||
'timestamp': ts,
|
||||
}
|
||||
|
||||
# Request post data (may be None)
|
||||
try:
|
||||
post = req.post_data
|
||||
info['request_post_data'] = post
|
||||
except Exception:
|
||||
try:
|
||||
info['request_post_data'] = req.post_data_text()
|
||||
except Exception:
|
||||
info['request_post_data'] = None
|
||||
|
||||
# Response body
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
try:
|
||||
# prefer text for textual content
|
||||
if 'application/json' in ctype or 'text/' in ctype or 'application/javascript' in ctype:
|
||||
txt = resp.text()
|
||||
info['response_text'] = txt
|
||||
info['response_body_file'] = None
|
||||
else:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
# fallback: save binary
|
||||
try:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
logging.exception('Failed to read response body for %s', req.url)
|
||||
|
||||
fname = f"{sanitize_filename(req.url)}_{ts}.json"
|
||||
out_path = os.path.join(args.output, fname)
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
import json
|
||||
|
||||
json.dump(info, fh, indent=2)
|
||||
|
||||
logging.info('Captured XHR: %s -> %s', req.url, out_path)
|
||||
captured.append(out_path)
|
||||
except Exception:
|
||||
logging.exception('Error in on_response')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
logging.info('Open the browser, log in if needed, and navigate to the page that triggers XHRs. Then return here and press Enter.')
|
||||
page.goto(args.topics_url)
|
||||
input('Press Enter to start capture (will reload the current page)...')
|
||||
|
||||
logging.info('Reloading page to trigger XHRs...')
|
||||
page.reload()
|
||||
|
||||
try:
|
||||
time.sleep(args.wait)
|
||||
logging.info('Waiting finished. You can interact to trigger more XHRs; press Ctrl+C when done.')
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Finishing capture...')
|
||||
finally:
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
logging.info('Done. Captured %d XHRs. Files saved under %s', len(captured), args.output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture-after-login helper
|
||||
|
||||
Usage:
|
||||
1. Run this script. It will open a visible browser with a persistent profile (so you can log in manually).
|
||||
2. Log in to STATdx in the opened browser and navigate to the topics page normally.
|
||||
3. Switch back to the terminal and press Enter. The script will reload the topics URL to trigger API calls,
|
||||
capture XHR/fetch response bodies, save them under the `output` directory, and dump cookies and localStorage.
|
||||
|
||||
Example:
|
||||
python scrapers/statdx_capture_after_login.py --profile .playwright_profile --output captured --domain app.statdx.com --topics-url https://app.statdx.com/
|
||||
|
||||
This is intended to capture the API JSON that the client uses to render pages after authentication.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import json
|
||||
import hashlib
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import urlparse
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
|
||||
def dump_storage_and_cookies(context, page, out_dir: str):
|
||||
try:
|
||||
cookies = context.cookies()
|
||||
with open(os.path.join(out_dir, 'cookies.json'), 'w', encoding='utf-8') as fh:
|
||||
json.dump(cookies, fh, indent=2)
|
||||
logging.info('Saved cookies to cookies.json')
|
||||
except Exception:
|
||||
logging.exception('Failed to dump cookies')
|
||||
|
||||
try:
|
||||
local = page.evaluate('()=> JSON.stringify(localStorage)')
|
||||
session = page.evaluate('()=> JSON.stringify(sessionStorage)')
|
||||
with open(os.path.join(out_dir, 'localStorage.json'), 'w', encoding='utf-8') as fh:
|
||||
fh.write(local)
|
||||
with open(os.path.join(out_dir, 'sessionStorage.json'), 'w', encoding='utf-8') as fh:
|
||||
fh.write(session)
|
||||
logging.info('Saved localStorage and sessionStorage')
|
||||
except Exception:
|
||||
logging.exception('Failed to dump storage')
|
||||
|
||||
|
||||
def save_response_body(resp, out_dir: str):
|
||||
try:
|
||||
url = resp.url
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
fname = sanitize_filename(url)
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
if 'application/json' in ctype or url.endswith('.json'):
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.json")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved JSON response: %s', path)
|
||||
return True
|
||||
elif 'text/html' in ctype or resp.request.resource_type == 'document':
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.html")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved HTML response: %s', path)
|
||||
return True
|
||||
elif ctype.startswith('text/'):
|
||||
try:
|
||||
text = resp.text()
|
||||
except Exception:
|
||||
text = resp.body().decode('utf-8', errors='replace')
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}.txt")
|
||||
with open(path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(text)
|
||||
logging.info('Saved text response: %s', path)
|
||||
return True
|
||||
else:
|
||||
# Save binary content (images, fonts, etc.)
|
||||
try:
|
||||
body = resp.body()
|
||||
path = os.path.join(out_dir, f"{fname}_{ts}")
|
||||
# try to add extension from content-type
|
||||
ctype_main = resp.headers.get('content-type', '').split(';')[0]
|
||||
try:
|
||||
import mimetypes
|
||||
|
||||
ext = mimetypes.guess_extension(ctype_main) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
if ext:
|
||||
path += ext
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
logging.info('Saved binary response: %s', path)
|
||||
return True
|
||||
except Exception:
|
||||
logging.exception('Failed to save binary response')
|
||||
return False
|
||||
except Exception:
|
||||
logging.exception('Failed to save response body')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile')
|
||||
parser.add_argument('--output', '-o', default='captured')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--topics-url', default='https://app.statdx.com/')
|
||||
parser.add_argument('--wait', type=float, default=8.0, help='Seconds to wait after reload to capture responses')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium
|
||||
logging.info('Launching persistent context with profile: %s (channel=%s)', args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False, channel=args.channel)
|
||||
else:
|
||||
context = browser.launch_persistent_context(user_data_dir=args.profile, headless=False)
|
||||
page = context.new_page()
|
||||
|
||||
captured = []
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
if args.domain not in resp.url:
|
||||
return
|
||||
rtype = resp.request.resource_type
|
||||
# focus on XHR/fetch and document
|
||||
if rtype in ('xhr', 'fetch', 'document'):
|
||||
ok = save_response_body(resp, args.output)
|
||||
if ok:
|
||||
captured.append(resp.url)
|
||||
except Exception:
|
||||
logging.exception('response handler error')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
logging.info('Open browser and log in if needed. Navigate to the topics page. Then return to the terminal and press Enter to trigger capture.')
|
||||
page.goto(args.topics_url)
|
||||
|
||||
input('Press Enter when you are logged in and on the page you want to capture...')
|
||||
|
||||
# Reload to trigger API calls while our handler is active
|
||||
logging.info('Reloading topics URL to trigger API calls...')
|
||||
page.reload()
|
||||
|
||||
# Optionally auto-scroll to trigger lazy loads
|
||||
try:
|
||||
page.evaluate(
|
||||
"""
|
||||
async () => {
|
||||
const wait = ms => new Promise(r => setTimeout(r, ms));
|
||||
const doc = document.scrollingElement || document.documentElement;
|
||||
let total = 0;
|
||||
while (total < doc.scrollHeight) {
|
||||
doc.scrollBy(0, 400);
|
||||
total += 400;
|
||||
await wait(150);
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
except Exception:
|
||||
logging.exception('auto-scroll failed during capture')
|
||||
|
||||
logging.info('Waiting %.1f seconds for responses...', args.wait)
|
||||
time.sleep(args.wait)
|
||||
|
||||
dump_storage_and_cookies(context, page, args.output)
|
||||
|
||||
# Save a final snapshot of the rendered page
|
||||
try:
|
||||
html = page.content()
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
fname = sanitize_filename(page.url)
|
||||
snap_path = os.path.join(args.output, f'snapshot_after_login_{fname}_{ts}.html')
|
||||
with open(snap_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(html)
|
||||
logging.info('Saved final page snapshot: %s', snap_path)
|
||||
# screenshots disabled per user request
|
||||
except Exception:
|
||||
logging.exception('Failed to save final snapshot')
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
logging.info('Done. Captured %d responses.', len(captured))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Diagnostic capture script: navigates to the site headful, records console/page errors, and logs requests and DNS checks.
|
||||
|
||||
Run:
|
||||
python scrapers/statdx_debug_capture.py --domain app.statdx.com --output debug_report.txt
|
||||
|
||||
It will open a visible browser; interact briefly, then press Ctrl+C in the terminal to finish and write the report.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import socket
|
||||
import time
|
||||
from datetime import datetime
|
||||
from datetime import timezone
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def dns_check(host):
|
||||
try:
|
||||
info = socket.getaddrinfo(host, None)
|
||||
addrs = sorted({i[4][0] for i in info})
|
||||
return True, addrs
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
||||
parser.add_argument('--output', '-o', default='debug_report.txt')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
report = []
|
||||
report.append(f'Report generated: {datetime.now(timezone.utc).isoformat()}Z')
|
||||
report.append(f'Domain: {args.domain}')
|
||||
|
||||
# DNS check for the main domain and some common vendors
|
||||
hosts = [args.domain, 'app.pendo.io', 'static.cloudflareinsights.com']
|
||||
report.append('\nDNS checks:')
|
||||
for h in hosts:
|
||||
ok, res = dns_check(h)
|
||||
report.append(f' - {h}: {ok} -> {res}')
|
||||
|
||||
requests = []
|
||||
console_logs = []
|
||||
page_errors = []
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=args.headless)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
def on_request(req):
|
||||
requests.append({'url': req.url, 'method': req.method, 'resource_type': req.resource_type})
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
requests.append({'url': resp.url, 'status': resp.status, 'resource_type': resp.request.resource_type})
|
||||
except Exception:
|
||||
requests.append({'url': getattr(resp, 'url', '<unknown>'), 'status': 'error'})
|
||||
|
||||
def on_console(msg):
|
||||
console_logs.append({'type': msg.type, 'text': msg.text})
|
||||
|
||||
def on_page_error(err):
|
||||
page_errors.append(str(err))
|
||||
|
||||
page.on('request', on_request)
|
||||
page.on('response', on_response)
|
||||
page.on('console', on_console)
|
||||
page.on('pageerror', on_page_error)
|
||||
|
||||
logging.info('Opening https://%s/', args.domain)
|
||||
page.goto(f'https://{args.domain}/')
|
||||
logging.info('Interact with the page now. Press Ctrl+C in the terminal to finish and write the report.')
|
||||
|
||||
try:
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Finishing and writing report...')
|
||||
finally:
|
||||
try:
|
||||
browser.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing browser (driver may have disconnected) — continuing to write report')
|
||||
|
||||
report.append('\nConsole logs:')
|
||||
for c in console_logs:
|
||||
report.append(f" - [{c['type']}] {c['text']}")
|
||||
|
||||
report.append('\nPage errors:')
|
||||
for e in page_errors:
|
||||
report.append(' - ' + e)
|
||||
|
||||
report.append('\nRecent requests (last 100):')
|
||||
for r in requests[-100:]:
|
||||
if isinstance(r, dict):
|
||||
report.append(' - {status} {resource_type} {url}'.format(status=r.get('status', ''), resource_type=r.get('resource_type', ''), url=r.get('url')))
|
||||
else:
|
||||
report.append(' - ' + str(r))
|
||||
|
||||
with open(args.output, 'w', encoding='utf-8') as fh:
|
||||
fh.write('\n'.join(report))
|
||||
|
||||
print('Wrote report to', args.output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live-capture browser: open a visible Chromium window, persist the profile, and save HTML/JSON responses
|
||||
as you browse so you can "download content as you browse normally".
|
||||
|
||||
Usage:
|
||||
- Install Playwright and browsers: python -m pip install playwright python-dotenv; python -m playwright install
|
||||
- Run:
|
||||
python scrapers/statdx_live_capture.py --output captured --domain app.statdx.com
|
||||
|
||||
What it does:
|
||||
- Launches a persistent Chromium context (profile saved to .playwright_profile by default) so cookies/session persist.
|
||||
- Opens a visible browser (headful). You can interact with it manually.
|
||||
- Listens to network responses. When it sees a response whose content-type is HTML or JSON (or document resource),
|
||||
it saves the response body to the `output` directory with a sanitized filename derived from the URL.
|
||||
|
||||
Notes / limitations:
|
||||
- This saves HTTP responses as they arrive. It does not run playback for dynamic client-rendered content (but it
|
||||
will save the HTML shell and any XHR/JSON responses used by the client).
|
||||
- You may need to filter by domain to avoid saving lots of third-party assets.
|
||||
- Respect terms of service and only use for accounts you own/have permission to access.
|
||||
"""
|
||||
|
||||
import os
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
import threading
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from urllib.parse import urlparse
|
||||
from dotenv import load_dotenv
|
||||
import mimetypes
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
"""Create a short filesystem-safe filename from a URL."""
|
||||
parsed = urlparse(url)
|
||||
# use host + path + query, but keep it short
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += "?" + parsed.query
|
||||
# remove leading/trailing slashes
|
||||
base = base.strip("/")
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
# replace problematic chars
|
||||
safe = "".join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
# add a short hash to avoid collisions and trim length
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
filename = f"{safe[:240]}_{h}"
|
||||
return filename
|
||||
|
||||
|
||||
def auto_scroll(page, step_ms: int = 50, step_px: int = 300):
|
||||
"""Scroll the page to the bottom slowly to trigger lazy loading.
|
||||
|
||||
Note: executed in the sync Playwright context by calling page.evaluate.
|
||||
"""
|
||||
try:
|
||||
page.evaluate(
|
||||
"""
|
||||
async (step_px, step_ms) => {
|
||||
const wait = (ms) => new Promise(r => setTimeout(r, ms));
|
||||
const doc = document.scrollingElement || document.documentElement;
|
||||
let total = 0;
|
||||
while (total < doc.scrollHeight) {
|
||||
doc.scrollBy(0, step_px);
|
||||
total += step_px;
|
||||
await wait(step_ms);
|
||||
}
|
||||
}
|
||||
""",
|
||||
step_px,
|
||||
step_ms,
|
||||
)
|
||||
except Exception:
|
||||
logging.exception('auto_scroll failed')
|
||||
|
||||
|
||||
def save_response(resp, out_dir: str):
|
||||
try:
|
||||
url = resp.url
|
||||
headers = resp.headers
|
||||
ctype = headers.get('content-type', '')
|
||||
|
||||
# Decide whether to save: HTML or JSON (or document)
|
||||
save_as_text = False
|
||||
save_as_binary = False
|
||||
|
||||
if 'text/html' in ctype or resp.request.resource_type == 'document':
|
||||
save_as_text = True
|
||||
ext = '.html'
|
||||
elif 'application/json' in ctype or 'text/javascript' in ctype or 'application/javascript' in ctype:
|
||||
save_as_text = True
|
||||
ext = '.json'
|
||||
elif ctype.startswith('text/'):
|
||||
save_as_text = True
|
||||
ext = '.txt'
|
||||
else:
|
||||
# treat as binary and save (images, fonts, video, etc.)
|
||||
save_as_binary = True
|
||||
ext = ''
|
||||
|
||||
if not (save_as_text or save_as_binary):
|
||||
return False
|
||||
|
||||
filename_base = sanitize_filename(url)
|
||||
out_path = os.path.join(out_dir, filename_base + ext)
|
||||
|
||||
logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0])
|
||||
|
||||
if save_as_text:
|
||||
try:
|
||||
body = resp.text()
|
||||
except Exception:
|
||||
# fallback to binary then decode
|
||||
body = resp.body().decode('utf-8', errors='replace')
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(body)
|
||||
else:
|
||||
body = resp.body()
|
||||
# try to guess extension from content-type
|
||||
try:
|
||||
ctype_main = ctype.split(';')[0]
|
||||
ext_guess = mimetypes.guess_extension(ctype_main) or ''
|
||||
except Exception:
|
||||
ext_guess = ''
|
||||
out_path = out_path + (ext_guess if ext_guess else '')
|
||||
with open(out_path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logging.exception('Failed to save response: %s', e)
|
||||
return False
|
||||
|
||||
|
||||
def save_page_snapshot(page, out_dir: str, reason: str = ''):
|
||||
try:
|
||||
url = page.url
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
filename_base = sanitize_filename(url)
|
||||
safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40]
|
||||
out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html")
|
||||
logging.info('Saving page snapshot: %s -> %s', url, out_path)
|
||||
content = page.content()
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
fh.write(content)
|
||||
# Screenshots are disabled (not saving visual snapshots per user request)
|
||||
return True
|
||||
except Exception:
|
||||
logging.exception('Failed to save page snapshot')
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright')
|
||||
parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses')
|
||||
parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory')
|
||||
parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain')
|
||||
parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)')
|
||||
parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot')
|
||||
parser.add_argument('--devtools', action='store_true', help='Open browser with devtools panel')
|
||||
parser.add_argument('--width', type=int, default=1280, help='Viewport width')
|
||||
parser.add_argument('--height', type=int, default=1600, help='Viewport height')
|
||||
parser.add_argument('--slowmo', type=int, default=0, help='Slow down Playwright actions (ms)')
|
||||
parser.add_argument('--user-agent', dest='user_agent', help='Custom User-Agent header')
|
||||
parser.add_argument('--wait-for-selector', dest='wait_for_selector', help='CSS selector to wait for after navigation')
|
||||
parser.add_argument('--auto-scroll', action='store_true', help='Automatically scroll the page when snapshotting to trigger lazy loads')
|
||||
parser.add_argument('--channel', default='chrome', help='Playwright browser channel to use (e.g. chrome). Set to empty to use bundled Chromium')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
# Use persistent context so cookies/localStorage persist between runs
|
||||
browser_type = p.chromium
|
||||
logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel)
|
||||
if args.channel:
|
||||
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel)
|
||||
else:
|
||||
context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
# Optionally you can set viewport, user agent, etc. here.
|
||||
# Create a new page with the requested viewport
|
||||
page = context.new_page()
|
||||
try:
|
||||
page.set_viewport_size({"width": args.width, "height": args.height})
|
||||
except Exception:
|
||||
# Older playwright versions may not support set_viewport_size on persistent contexts
|
||||
pass
|
||||
|
||||
def handle_response(resp):
|
||||
try:
|
||||
if args.domain not in resp.url:
|
||||
return
|
||||
# save textual responses
|
||||
saved = save_response(resp, args.output)
|
||||
# If this response is a document or an XHR/fetch that likely carried data,
|
||||
# also save a rendered page snapshot so client-side content is captured.
|
||||
rtype = resp.request.resource_type
|
||||
if saved and rtype in ("document", "xhr", "fetch"):
|
||||
# page may not have finished rendering; give it a small chance to update
|
||||
try:
|
||||
time.sleep(args.snapshot_delay)
|
||||
if args.auto_scroll:
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason=rtype)
|
||||
except Exception:
|
||||
logging.exception('snapshot after response failed')
|
||||
except Exception:
|
||||
logging.exception('Error in response handler')
|
||||
|
||||
|
||||
page.on('response', handle_response)
|
||||
|
||||
# Log console messages and page errors to help diagnose missing content
|
||||
def on_console(msg):
|
||||
try:
|
||||
logging.info('PAGE LOG [%s] %s', msg.type, msg.text)
|
||||
except Exception:
|
||||
logging.info('PAGE LOG: %s', msg)
|
||||
|
||||
def on_page_error(exc):
|
||||
logging.error('PAGE ERROR: %s', exc)
|
||||
|
||||
page.on('console', on_console)
|
||||
page.on('pageerror', on_page_error)
|
||||
|
||||
logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output)
|
||||
logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).')
|
||||
logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.')
|
||||
|
||||
# Navigate to the domain root so you can log in manually if needed
|
||||
start_url = f'https://{args.domain}/'
|
||||
page.goto(start_url)
|
||||
# Optionally wait for a selector that indicates the page finished rendering
|
||||
if args.wait_for_selector:
|
||||
try:
|
||||
logging.info('Waiting for selector: %s', args.wait_for_selector)
|
||||
page.wait_for_selector(args.wait_for_selector, timeout=20000)
|
||||
except Exception:
|
||||
logging.exception('Waiting for selector timed out')
|
||||
|
||||
def stdin_watcher():
|
||||
# Run in a background thread so the main thread can keep the browser running
|
||||
logging.debug('stdin watcher started')
|
||||
for line in sys.stdin:
|
||||
cmd = line.strip().lower()
|
||||
if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save':
|
||||
logging.info('Manual snapshot requested via stdin')
|
||||
try:
|
||||
if args.auto_scroll:
|
||||
try:
|
||||
auto_scroll(page)
|
||||
except Exception:
|
||||
logging.exception('manual auto-scroll failed')
|
||||
save_page_snapshot(page, args.output, reason='manual')
|
||||
except Exception:
|
||||
logging.exception('manual snapshot failed')
|
||||
|
||||
watcher = threading.Thread(target=stdin_watcher, daemon=True)
|
||||
watcher.start()
|
||||
|
||||
try:
|
||||
# Keep the script running while the user interacts with the headed browser
|
||||
while True:
|
||||
time.sleep(1)
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Shutting down...')
|
||||
finally:
|
||||
context.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Launch the site in an actual installed Chrome via Playwright's 'chrome' channel.
|
||||
|
||||
This can help if the headful Playwright Chromium environment differs from your regular Chrome and the site
|
||||
only fully renders in a real browser environment.
|
||||
|
||||
Usage:
|
||||
python scrapers/statdx_play_chrome.py --profile .chrome_profile --url https://app.statdx.com/ --screenshot out.png
|
||||
|
||||
Notes:
|
||||
- Requires Playwright and that the 'chrome' browser is available on your machine.
|
||||
- If Playwright cannot find the channel, install Chrome or use the system browser path.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
from time import sleep
|
||||
from urllib.parse import urlparse
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from datetime import datetime, timezone
|
||||
from playwright.sync_api import sync_playwright
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env in the repository root (if present)
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--profile', '-p', default='.chrome_profile')
|
||||
parser.add_argument('--url', '-u', default='https://app.statdx.com/')
|
||||
parser.add_argument('--domain', '-d', default='app.statdx.com', help='Domain to filter captures')
|
||||
parser.add_argument('--screenshot', '-s', default='chrome_page.png')
|
||||
parser.add_argument('--wait-for', help='Optional selector to wait for before screenshot')
|
||||
parser.add_argument('--headless', action='store_true')
|
||||
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
|
||||
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
|
||||
parser.add_argument('--post-login-selector', default='#ds-app', help='Selector indicating successful login (default: #ds-app)')
|
||||
parser.add_argument('--wait-after-login', type=float, default=3.0, help='Seconds to wait after login before snapshot')
|
||||
parser.add_argument('--keep-open', action='store_true', help='Leave the browser open after actions')
|
||||
parser.add_argument('--capture', action='store_true', help='Capture XHR/fetch requests and responses while the browser is open')
|
||||
parser.add_argument('--capture-output', default='xhr_captured', help='Directory to save captured XHRs')
|
||||
parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait after reload for XHRs to complete when capture starts')
|
||||
args = parser.parse_args()
|
||||
|
||||
os.makedirs(args.profile, exist_ok=True)
|
||||
|
||||
with sync_playwright() as p:
|
||||
try:
|
||||
logging.info('Launching Chrome channel with profile %s (headless=%s)', args.profile, args.headless)
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, channel='chrome', headless=args.headless)
|
||||
except Exception as e:
|
||||
logging.error('Failed to launch Chrome channel: %s', e)
|
||||
logging.info('Falling back to default Chromium persistent context')
|
||||
context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless)
|
||||
|
||||
page = context.new_page()
|
||||
page.goto(args.url)
|
||||
|
||||
# Setup capture if requested
|
||||
captured = []
|
||||
if args.capture:
|
||||
os.makedirs(args.capture_output, exist_ok=True)
|
||||
|
||||
def sanitize_filename(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
base = parsed.netloc + parsed.path
|
||||
if parsed.query:
|
||||
base += '?' + parsed.query
|
||||
base = base.strip('/')
|
||||
if not base:
|
||||
base = parsed.netloc
|
||||
safe = ''.join(c if c.isalnum() or c in ('-', '_', '.') else '_' for c in base)
|
||||
h = hashlib.sha1(url.encode('utf-8')).hexdigest()[:8]
|
||||
return f"{safe[:200]}_{h}"
|
||||
|
||||
def save_binary(body: bytes, out_dir: str, url: str, ts: str, ctype: str):
|
||||
ext = ''
|
||||
try:
|
||||
ext = mimetypes.guess_extension(ctype.split(';')[0]) or ''
|
||||
except Exception:
|
||||
ext = ''
|
||||
fname = f"{sanitize_filename(url)}_{ts}{ext}"
|
||||
path = os.path.join(out_dir, fname)
|
||||
with open(path, 'wb') as fh:
|
||||
fh.write(body)
|
||||
return path
|
||||
|
||||
def on_response(resp):
|
||||
try:
|
||||
req = resp.request
|
||||
rtype = req.resource_type
|
||||
if args.domain not in req.url:
|
||||
return
|
||||
if rtype not in ('xhr', 'fetch'):
|
||||
return
|
||||
|
||||
ts = datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
|
||||
|
||||
info = {
|
||||
'url': req.url,
|
||||
'method': req.method,
|
||||
'resource_type': rtype,
|
||||
'status': resp.status,
|
||||
'request_headers': dict(req.headers),
|
||||
'response_headers': dict(resp.headers),
|
||||
'timestamp': ts,
|
||||
}
|
||||
|
||||
try:
|
||||
post = req.post_data
|
||||
info['request_post_data'] = post
|
||||
except Exception:
|
||||
try:
|
||||
info['request_post_data'] = req.post_data_text()
|
||||
except Exception:
|
||||
info['request_post_data'] = None
|
||||
|
||||
ctype = resp.headers.get('content-type', '')
|
||||
try:
|
||||
if 'application/json' in ctype or 'text/' in ctype or 'application/javascript' in ctype:
|
||||
txt = resp.text()
|
||||
info['response_text'] = txt
|
||||
info['response_body_file'] = None
|
||||
else:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
try:
|
||||
body = resp.body()
|
||||
path = save_binary(body, args.capture_output, req.url, ts, ctype)
|
||||
info['response_text'] = None
|
||||
info['response_body_file'] = path
|
||||
except Exception:
|
||||
logging.exception('Failed to read response body for %s', req.url)
|
||||
|
||||
fname = f"{sanitize_filename(req.url)}_{ts}.json"
|
||||
out_path = os.path.join(args.capture_output, fname)
|
||||
with open(out_path, 'w', encoding='utf-8') as fh:
|
||||
import json
|
||||
|
||||
json.dump(info, fh, indent=2)
|
||||
|
||||
logging.info('Captured XHR: %s -> %s', req.url, out_path)
|
||||
captured.append(out_path)
|
||||
except Exception:
|
||||
logging.exception('Error in capture on_response')
|
||||
|
||||
page.on('response', on_response)
|
||||
|
||||
# Automatic login if credentials provided
|
||||
username = args.username or os.getenv('STATDX_USERNAME')
|
||||
password = args.password or os.getenv('STATDX_PASSWORD')
|
||||
if username and password:
|
||||
logging.info('Attempting automatic login using provided credentials')
|
||||
# candidate selectors for username and password fields
|
||||
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
|
||||
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
|
||||
|
||||
user_sel = None
|
||||
for sel in user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
pass_sel = None
|
||||
for sel in pass_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
pass_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel or not pass_sel:
|
||||
logging.warning('Could not auto-detect username/password fields. Skipping automatic login.')
|
||||
else:
|
||||
try:
|
||||
page.fill(user_sel, username)
|
||||
sleep(0.1)
|
||||
page.fill(pass_sel, password)
|
||||
sleep(0.1)
|
||||
# try submit buttons
|
||||
submit_selectors = ['button.primary.submitSelector', 'button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
|
||||
clicked = False
|
||||
for s in submit_selectors:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
logging.info('Clicked submit button using selector: %s', s)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
if not clicked:
|
||||
logging.info('No submit button clicked; pressing Enter in password field')
|
||||
page.press(pass_sel, 'Enter')
|
||||
|
||||
# wait for either navigation or a post-login selector
|
||||
try:
|
||||
if args.post_login_selector:
|
||||
page.wait_for_selector(args.post_login_selector, timeout=int(args.wait_after_login * 1000))
|
||||
except Exception:
|
||||
logging.info('Post-login selector not found within timeout')
|
||||
|
||||
# small extra wait to let JS render
|
||||
sleep(args.wait_after_login)
|
||||
except Exception:
|
||||
logging.exception('Automatic login attempt failed')
|
||||
|
||||
if args.wait_for:
|
||||
try:
|
||||
logging.info('Waiting for selector: %s', args.wait_for)
|
||||
page.wait_for_selector(args.wait_for, timeout=20000)
|
||||
except Exception:
|
||||
logging.exception('wait_for selector timed out')
|
||||
|
||||
# give some time for JS to finish
|
||||
page.wait_for_timeout(1500)
|
||||
|
||||
# Screenshots are disabled per user request
|
||||
|
||||
if args.keep_open:
|
||||
logging.info('Leaving browser open for manual inspection. Close the browser window to finish.')
|
||||
try:
|
||||
# keep running until user closes browser
|
||||
context.wait_for_event('close')
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
context.close()
|
||||
except Exception:
|
||||
logging.exception('Error closing context')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Playwright-based login + scraping for STATdx (recommended for JS-heavy sites).
|
||||
|
||||
This script launches a headless Chromium, navigates to the app, fills username/password, clicks the submit button,
|
||||
waits for a post-login selector, then saves the resulting HTML to disk.
|
||||
|
||||
You may need to tweak selectors if the login form uses unusual markup or an SSO redirect.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
USERNAME = os.getenv("STATDX_USERNAME")
|
||||
PASSWORD = os.getenv("STATDX_PASSWORD")
|
||||
TARGET_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
||||
|
||||
if not USERNAME or not PASSWORD:
|
||||
raise SystemExit("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
|
||||
def run(headless=True, timeout=30000):
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=headless)
|
||||
context = browser.new_context()
|
||||
page = context.new_page()
|
||||
|
||||
logging.info("Opening %s", TARGET_URL)
|
||||
page.goto(TARGET_URL, wait_until="domcontentloaded", timeout=timeout)
|
||||
|
||||
# Heuristics to find fields — adapt per the real login form
|
||||
password_selector = "input[type=\"password\"]"
|
||||
try:
|
||||
page.wait_for_selector(password_selector, timeout=8000)
|
||||
logging.info("Password field found")
|
||||
except Exception:
|
||||
logging.warning("Password field not found quickly; continuing to try common selectors")
|
||||
|
||||
# Try to detect username/email field by common selectors
|
||||
candidate_user_selectors = [
|
||||
"input[type=\"email\"]",
|
||||
"input[name*=\"email\" i]",
|
||||
"input[name*=\"user\" i]",
|
||||
"input[type=\"text\"]",
|
||||
]
|
||||
|
||||
user_sel = None
|
||||
for sel in candidate_user_selectors:
|
||||
try:
|
||||
if page.query_selector(sel):
|
||||
user_sel = sel
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not user_sel:
|
||||
logging.warning("Could not auto-detect username field; you may need to edit selectors in this script.")
|
||||
# fallback: pick first visible text input
|
||||
inputs = page.query_selector_all("input[type=\"text\"]")
|
||||
if inputs:
|
||||
user_sel = "input[type=\"text\"]"
|
||||
|
||||
logging.info("Using username selector: %s", user_sel)
|
||||
|
||||
if user_sel:
|
||||
page.fill(user_sel, USERNAME)
|
||||
else:
|
||||
logging.error("No username field found. Aborting.")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
# Fill password
|
||||
if page.query_selector(password_selector):
|
||||
page.fill(password_selector, PASSWORD)
|
||||
else:
|
||||
logging.error("No password field found. Aborting.")
|
||||
browser.close()
|
||||
return
|
||||
|
||||
# Try to click a submit button
|
||||
submit_selectors = [
|
||||
"button[type=submit]",
|
||||
"input[type=submit]",
|
||||
"button:has-text(\"Sign in\")",
|
||||
"button:has-text(\"Sign In\")",
|
||||
"button:has-text(\"Log in\")",
|
||||
"button:has-text(\"Log In\")",
|
||||
]
|
||||
|
||||
clicked = False
|
||||
for s in submit_selectors:
|
||||
try:
|
||||
btn = page.query_selector(s)
|
||||
if btn:
|
||||
btn.click()
|
||||
clicked = True
|
||||
logging.info("Clicked submit via selector %s", s)
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if not clicked:
|
||||
logging.warning("No submit button clicked; attempting to press Enter on password field")
|
||||
page.press(password_selector, "Enter")
|
||||
|
||||
# Wait for navigation or the app root element that indicates successful login
|
||||
try:
|
||||
page.wait_for_selector("#ds-app", timeout=20000)
|
||||
logging.info("#ds-app found — likely logged in and app rendered")
|
||||
except Exception:
|
||||
logging.info("Timed out waiting for #ds-app — page content may still be loading or login failed.")
|
||||
|
||||
# Optional: wait a bit for client-side rendering
|
||||
time.sleep(2)
|
||||
|
||||
content = page.content()
|
||||
out_file = "topics_playwright.html"
|
||||
with open(out_file, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
logging.info("Saved rendered HTML to %s", out_file)
|
||||
|
||||
browser.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(headless=True)
|
||||
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Form-based login + scraping example for STATdx (best-effort).
|
||||
|
||||
This script attempts to:
|
||||
- GET the login page
|
||||
- detect a login form and its fields (username/password)
|
||||
- submit credentials and preserve the session cookie
|
||||
- fetch the target topics page and print/save HTML
|
||||
|
||||
Notes:
|
||||
- Most modern sites use JS-driven auth or SSO; if this fails, use the Playwright script.
|
||||
- Inspect the login POST URL and field names with your browser and set the environment overrides in .env
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
|
||||
# Configuration via environment (or fallbacks)
|
||||
LOGIN_PAGE = os.getenv("STATDX_LOGIN_URL", "https://app.statdx.com/")
|
||||
TOPICS_URL = os.getenv("STATDX_TOPICS_URL", "https://app.statdx.com/")
|
||||
USERNAME = os.getenv("STATDX_USERNAME")
|
||||
PASSWORD = os.getenv("STATDX_PASSWORD")
|
||||
OVERRIDE_USER_FIELD = os.getenv("STATDX_USERNAME_FIELD")
|
||||
OVERRIDE_PASS_FIELD = os.getenv("STATDX_PASSWORD_FIELD")
|
||||
|
||||
if not USERNAME or not PASSWORD:
|
||||
logging.error("Please set STATDX_USERNAME and STATDX_PASSWORD in your environment or .env file.")
|
||||
sys.exit(1)
|
||||
|
||||
session = requests.Session()
|
||||
|
||||
|
||||
def find_login_form(html, base_url):
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
forms = soup.find_all("form")
|
||||
if not forms:
|
||||
return None
|
||||
|
||||
# Heuristic: pick the form that contains an input of type password
|
||||
for form in forms:
|
||||
if form.find("input", {"type": "password"}):
|
||||
action = form.get("action") or base_url
|
||||
method = (form.get("method") or "get").lower()
|
||||
inputs = {}
|
||||
for inp in form.find_all("input"):
|
||||
name = inp.get("name")
|
||||
if not name:
|
||||
continue
|
||||
value = inp.get("value", "")
|
||||
inputs[name] = value
|
||||
return {"action": urljoin(base_url, action), "method": method, "inputs": inputs}
|
||||
return None
|
||||
|
||||
|
||||
def guess_field_names(inputs):
|
||||
# Try to find which input names likely correspond to username and password
|
||||
u_name = None
|
||||
p_name = None
|
||||
for name in inputs:
|
||||
ln = name.lower()
|
||||
if any(x in ln for x in ("user", "email", "login")) and not u_name:
|
||||
u_name = name
|
||||
if "pass" in ln and not p_name:
|
||||
p_name = name
|
||||
# Fallback to common names
|
||||
if not u_name:
|
||||
for cand in ("username", "email", "user"):
|
||||
if cand in inputs:
|
||||
u_name = cand
|
||||
break
|
||||
if not p_name:
|
||||
for cand in ("password", "pass"):
|
||||
if cand in inputs:
|
||||
p_name = cand
|
||||
break
|
||||
return u_name, p_name
|
||||
|
||||
|
||||
def login_using_form(login_page_url):
|
||||
logging.info("Fetching login page: %s", login_page_url)
|
||||
r = session.get(login_page_url, allow_redirects=True)
|
||||
r.raise_for_status()
|
||||
form = find_login_form(r.text, r.url)
|
||||
if not form:
|
||||
logging.warning("No login form found on page. The site might be JS-driven. Consider the Playwright script.")
|
||||
return False
|
||||
|
||||
action = form["action"]
|
||||
method = form["method"]
|
||||
inputs = form["inputs"].copy()
|
||||
|
||||
# Determine username / password field names
|
||||
user_field = OVERRIDE_USER_FIELD or guess_field_names(inputs)[0]
|
||||
pass_field = OVERRIDE_PASS_FIELD or guess_field_names(inputs)[1]
|
||||
|
||||
if not user_field or not pass_field:
|
||||
logging.error("Could not detect username/password field names. Please inspect the login form and set STATDX_USERNAME_FIELD and STATDX_PASSWORD_FIELD in .env")
|
||||
return False
|
||||
|
||||
inputs[user_field] = USERNAME
|
||||
inputs[pass_field] = PASSWORD
|
||||
|
||||
logging.info("Submitting login form to: %s", action)
|
||||
if method == "post":
|
||||
resp = session.post(action, data=inputs, allow_redirects=True)
|
||||
else:
|
||||
resp = session.get(action, params=inputs, allow_redirects=True)
|
||||
|
||||
logging.info("Login response: %s %s", resp.status_code, resp.url)
|
||||
|
||||
# Basic success check: did we land on a different URL or see a known element?
|
||||
if resp.url != login_page_url and resp.status_code in (200, 302, 303):
|
||||
logging.info("Looks like login may have succeeded (redirected).")
|
||||
return True
|
||||
|
||||
# Heuristic: search for 'logout' or 'sign out' in HTML
|
||||
if "logout" in resp.text.lower() or "sign out" in resp.text.lower():
|
||||
logging.info("Found logout text — login probably succeeded.")
|
||||
return True
|
||||
|
||||
logging.warning("Login may have failed (no redirect or logout text). Try Playwright or provide exact POST URL/fields.)")
|
||||
return False
|
||||
|
||||
|
||||
def fetch_topics(url):
|
||||
logging.info("Fetching topics page: %s", url)
|
||||
r = session.get(url)
|
||||
r.raise_for_status()
|
||||
return r.text
|
||||
|
||||
|
||||
def save_html(content, path="topics.html"):
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write(content)
|
||||
logging.info("Saved HTML to %s", path)
|
||||
|
||||
|
||||
def main():
|
||||
ok = login_using_form(LOGIN_PAGE)
|
||||
if not ok:
|
||||
logging.error("Form-based login failed. If the site uses JS/SSO, try the Playwright script: scrapers/statdx_playwright.py")
|
||||
return
|
||||
|
||||
html = fetch_topics(TOPICS_URL)
|
||||
save_html(html)
|
||||
# As a minimal parser example, show the contents of <div id="ds-app">
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
ds_app = soup.find(id="ds-app")
|
||||
if ds_app:
|
||||
logging.info("Found #ds-app — page likely relies on JavaScript. For rendered content use Playwright to get client-side-rendered HTML.")
|
||||
else:
|
||||
logging.info("#ds-app not found (or empty). You can parse the HTML above for static content.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"url": "https://app.statdx.com/csrf/token",
|
||||
"method": "GET",
|
||||
"resource_type": "xhr",
|
||||
"status": 200,
|
||||
"request_headers": {
|
||||
"sec-ch-ua-platform": "\"Linux\"",
|
||||
"referer": "https://app.statdx.com/",
|
||||
"x-requested-with": "XMLHttpRequest",
|
||||
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
||||
"accept": "*/*",
|
||||
"sec-ch-ua": "\"Not=A?Brand\";v=\"24\", \"Chromium\";v=\"140\"",
|
||||
"sec-ch-ua-mobile": "?0"
|
||||
},
|
||||
"response_headers": {
|
||||
"version-pss-dsjs-config": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
||||
"version-pss-dsjs": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
||||
"content-encoding": "gzip",
|
||||
"cf-cache-status": "DYNAMIC",
|
||||
"etag": "W/\"10c-vLpriK5k7wHbwVGauL6RBVcuBHQ\"",
|
||||
"x-ua-compatible": "IE=edge",
|
||||
"date": "Tue, 14 Oct 2025 12:41:21 GMT",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"vary": "Accept-Encoding",
|
||||
"version-pss-sparks-event-client": "2.1.1",
|
||||
"strict-transport-security": "max-age=31536000; includeSubDomains",
|
||||
"content-security-policy": "script-src 'self' 'unsafe-eval' 'unsafe-inline' js-agent.newrelic.com static.cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; style-src 'self' 'unsafe-inline' fonts.googleapis.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; img-src 'self' data: app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com *.cloudfront.net; media-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com *.cloudfront.net; connect-src 'self' ws: cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; frame-src 'self' app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com; worker-src 'self' blob: scopus.com service.elsevier.com pendo.reaxys.com; object-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com",
|
||||
"cache-control": "no-cache, no-store",
|
||||
"tdm-reservation": "1",
|
||||
"x-ratelimit-reset": "1760445742",
|
||||
"hostname": "sdxapp3",
|
||||
"cf-ray": "98e726878fefbea1-LHR",
|
||||
"x-ratelimit-remaining": "499",
|
||||
"version-pss-cmejs": "9.3.3",
|
||||
"tdm-policy": "https://www.elsevier.com/tdm/tdmrep-policy.json",
|
||||
"x-ratelimit-limit": "500",
|
||||
"version-pss-authenticationjs": "7.1.0",
|
||||
"x-powered-by": "Express",
|
||||
"server": "cloudflare"
|
||||
},
|
||||
"timestamp": "20251014T124122Z",
|
||||
"request_post_data": null,
|
||||
"response_text": "{\"token\":\"c67a4a30b300096e11483fc08f9dcb69e679ef1152816abe6bcad85c694959d5ba2e29cab29da9e13a743b29daac792a44a0af3bcf7f0b6476c1691d52b01918472529e208a3431dc22a435435c7947b1212f546ae3b618ad1701e1f37f616c6f6a76cfa179fb82706566692759ca11b617ba0d9501d9ee88a7e1e6ac2c8be39\"}",
|
||||
"response_body_file": null
|
||||
}
|
||||
Reference in New Issue
Block a user