Ross
491fca2f42
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
115 lines
3.7 KiB
Python
115 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Diagnostic capture script: navigates to the site headful, records console/page errors, and logs requests and DNS checks.
|
|
|
|
Run:
|
|
python scrapers/statdx_debug_capture.py --domain app.statdx.com --output debug_report.txt
|
|
|
|
It will open a visible browser; interact briefly, then press Ctrl+C in the terminal to finish and write the report.
|
|
"""
|
|
|
|
import argparse
|
|
import logging
|
|
import socket
|
|
import time
|
|
from datetime import datetime
|
|
from datetime import timezone
|
|
from playwright.sync_api import sync_playwright
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
|
|
|
|
|
def dns_check(host):
|
|
try:
|
|
info = socket.getaddrinfo(host, None)
|
|
addrs = sorted({i[4][0] for i in info})
|
|
return True, addrs
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--domain', '-d', default='app.statdx.com')
|
|
parser.add_argument('--output', '-o', default='debug_report.txt')
|
|
parser.add_argument('--headless', action='store_true')
|
|
args = parser.parse_args()
|
|
|
|
report = []
|
|
report.append(f'Report generated: {datetime.now(timezone.utc).isoformat()}Z')
|
|
report.append(f'Domain: {args.domain}')
|
|
|
|
# DNS check for the main domain and some common vendors
|
|
hosts = [args.domain, 'app.pendo.io', 'static.cloudflareinsights.com']
|
|
report.append('\nDNS checks:')
|
|
for h in hosts:
|
|
ok, res = dns_check(h)
|
|
report.append(f' - {h}: {ok} -> {res}')
|
|
|
|
requests = []
|
|
console_logs = []
|
|
page_errors = []
|
|
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=args.headless)
|
|
context = browser.new_context()
|
|
page = context.new_page()
|
|
|
|
def on_request(req):
|
|
requests.append({'url': req.url, 'method': req.method, 'resource_type': req.resource_type})
|
|
|
|
def on_response(resp):
|
|
try:
|
|
requests.append({'url': resp.url, 'status': resp.status, 'resource_type': resp.request.resource_type})
|
|
except Exception:
|
|
requests.append({'url': getattr(resp, 'url', '<unknown>'), 'status': 'error'})
|
|
|
|
def on_console(msg):
|
|
console_logs.append({'type': msg.type, 'text': msg.text})
|
|
|
|
def on_page_error(err):
|
|
page_errors.append(str(err))
|
|
|
|
page.on('request', on_request)
|
|
page.on('response', on_response)
|
|
page.on('console', on_console)
|
|
page.on('pageerror', on_page_error)
|
|
|
|
logging.info('Opening https://%s/', args.domain)
|
|
page.goto(f'https://{args.domain}/')
|
|
logging.info('Interact with the page now. Press Ctrl+C in the terminal to finish and write the report.')
|
|
|
|
try:
|
|
while True:
|
|
time.sleep(1)
|
|
except KeyboardInterrupt:
|
|
logging.info('Finishing and writing report...')
|
|
finally:
|
|
try:
|
|
browser.close()
|
|
except Exception:
|
|
logging.exception('Error closing browser (driver may have disconnected) — continuing to write report')
|
|
|
|
report.append('\nConsole logs:')
|
|
for c in console_logs:
|
|
report.append(f" - [{c['type']}] {c['text']}")
|
|
|
|
report.append('\nPage errors:')
|
|
for e in page_errors:
|
|
report.append(' - ' + e)
|
|
|
|
report.append('\nRecent requests (last 100):')
|
|
for r in requests[-100:]:
|
|
if isinstance(r, dict):
|
|
report.append(' - {status} {resource_type} {url}'.format(status=r.get('status', ''), resource_type=r.get('resource_type', ''), url=r.get('url')))
|
|
else:
|
|
report.append(' - ' + str(r))
|
|
|
|
with open(args.output, 'w', encoding='utf-8') as fh:
|
|
fh.write('\n'.join(report))
|
|
|
|
print('Wrote report to', args.output)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|