diff --git a/.gitignore b/.gitignore index ff790f1..589eb99 100644 --- a/.gitignore +++ b/.gitignore @@ -69,4 +69,5 @@ secrets.json playwright_profile captured/ -xhr_captured/ \ No newline at end of file +xhr_captured/ +xhr_captured_async/ \ No newline at end of file diff --git a/scrapers/capture_passive_playwright.py b/scrapers/capture_passive_playwright.py new file mode 100644 index 0000000..ff32562 --- /dev/null +++ b/scrapers/capture_passive_playwright.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +""" +Passive XHR/fetch capture using Playwright. + +This script launches a persistent browser profile (so you can log in), +optionally attempts auto-login, then attaches a response handler and saves +every XHR/fetch response body and metadata into `--output-dir` while you +browse. It is intentionally passive: it doesn't reload the page unless you +ask it to; it simply listens and saves responses as they happen. + +Usage: + python scrapers/capture_passive_playwright.py --output-dir xhr_captured --continuous + +Options: + --username/--password or STATDX_USERNAME/STATDX_PASSWORD via .env for auto-login + --no-prompt to start listening immediately + --capture-types to adjust which resource types to record (default: xhr,fetch) +""" +from pathlib import Path +import argparse +import json +import sys +import time +import mimetypes +import hashlib +from datetime import datetime, timezone +import os + +from loguru import logger + +from playwright.sync_api import sync_playwright +import asyncio +try: + from playwright._impl._errors import TargetClosedError as _PlaywrightTargetClosedError + TARGET_CLOSED_EXCEPTIONS = (_PlaywrightTargetClosedError,) +except Exception: + # If we cannot import Playwright internal error types, fall back to Exception + TARGET_CLOSED_EXCEPTIONS = (Exception,) +from dotenv import load_dotenv + +logger.remove() +logger.add("capture_passive_playwright.log", rotation="10 MB") +logger.add(sys.stderr, level="TRACE") +# Keep detailed logs in the file only to avoid noisy console output while you browse. +# We will print a short summary to stdout when the browser/context is closed. + +load_dotenv() + + +def sanitize(s: str) -> str: + out = [] + for ch in s: + if ch.isalnum() or ch in ('-', '_', '.'): + out.append(ch) + else: + out.append('_') + name = ''.join(out) + if len(name) > 180: + name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8] + return name + + +def guess_ext(ctype: str) -> str: + if not ctype: + return 'bin' + mt = ctype.split(';')[0].strip() + ext = mimetypes.guess_extension(mt) + if ext: + return ext.lstrip('.') + if mt.startswith('image/'): + return mt.split('/')[1] + if mt == 'application/json': + return 'json' + if mt == 'text/html': + return 'html' + if mt.startswith('text/'): + return 'txt' + return 'bin' + + +def now_ts(): + return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + + +def save_binary(path: Path, data: bytes): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'wb') as f: + f.write(data) + + +def save_text(path: Path, text: str): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + + +def try_pretty_json(s: str): + try: + parsed = json.loads(s) + return json.dumps(parsed, ensure_ascii=False, indent=2) + except Exception: + return None + + +def attempt_autologin(page, username, password, post_login_selector, wait_after=3.0): + if not username or not password: + return False + logger.info('Attempting auto-login') + user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]'] + pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector'] + + user_sel = None + for sel in user_selectors: + try: + if page.query_selector(sel): + user_sel = sel + break + except Exception: + continue + + pass_sel = None + for sel in pass_selectors: + try: + if page.query_selector(sel): + pass_sel = sel + break + except Exception: + continue + + if not user_sel or not pass_sel: + logger.warning('Could not find login fields for auto-login') + return False + + try: + page.fill(user_sel, username) + time.sleep(0.1) + page.fill(pass_sel, password) + time.sleep(0.1) + # try to submit + submit_try = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")'] + clicked = False + for s in submit_try: + try: + btn = page.query_selector(s) + if btn: + btn.click() + clicked = True + break + except Exception: + continue + if not clicked: + page.press(pass_sel, 'Enter') + + if post_login_selector: + try: + page.wait_for_selector(post_login_selector, timeout=10000) + except Exception: + logger.info('Post-login selector not found in timeout') + + time.sleep(wait_after) + return True + except Exception: + logger.exception('Auto-login failed') + return False + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--url', '-u', default='https://app.statdx.com/') + parser.add_argument('--profile', '-p', default='playwright_profile') + parser.add_argument('--output-dir', '-o', default='xhr_captured') + parser.add_argument('--headless', action='store_true') + parser.add_argument('--continuous', action='store_true', help='Keep listening and saving as you browse') + parser.add_argument('--no-prompt', action='store_true', help='Start listening immediately without waiting for Enter') + parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)') + parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)') + parser.add_argument('--post-login-selector', default='#ds-app') + parser.add_argument('--capture-types', default='xhr,fetch,document,other', help='Comma-separated resource types to capture (empty = all)') + args = parser.parse_args() + + output = Path(args.output_dir) + output.mkdir(parents=True, exist_ok=True) + index_path = output / 'capture_index.jsonl' + + username = args.username or os.getenv('STATDX_USERNAME') + password = args.password or os.getenv('STATDX_PASSWORD') + capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()} + + with sync_playwright() as p: + # launch persistent context + try: + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome') + except Exception: + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) + + page = context.new_page() + # NOTE: don't navigate until we've attached listeners so we don't miss + # early requests from the initial load. + + # response handler (define before we attach listeners) + def on_response(resp): + """Safe response handler: reads response bodies but tolerates browser/context close. + + Playwright can raise TargetClosedError or asyncio.CancelledError when the + browser or context closes while we're trying to read a body. Those are + normal during shutdown and should not produce noisy tracebacks. This + handler therefore catches those and returns quietly. + """ + logger.debug(f'Response: {getattr(resp, "status", "?")} {getattr(resp, "url", "?")}') + try: + req = resp.request + rtype = (req.resource_type or '').lower() + # Log every response minimally so user sees activity + logger.info('Response event: %s %s %s', getattr(resp, 'status', '?'), rtype, req.url) + + # write debug line to disk so we can verify events even if console is quiet + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\t{getattr(resp, 'status', '?')}\t{rtype}\t{req.url}\n") + except Exception: + pass + + # If capture_types is empty, capture everything. Otherwise check membership. + allow = False + if not capture_types: + allow = True + elif rtype in capture_types: + allow = True + # Heuristic: always capture document content endpoints we care about + if '/document/content/' in req.url or '/document/summary/' in req.url: + allow = True + if not allow: + return + + url = req.url + ts = now_ts() + status = getattr(resp, 'status', None) + req_headers = dict(req.headers) + resp_headers = dict(resp.headers) + + meta = { + 'url': url, + 'method': req.method, + 'resource_type': rtype, + 'status': status, + 'timestamp': ts, + 'request_headers': req_headers, + 'response_headers': resp_headers, + } + + # request post data if any + try: + post = req.post_data + meta['request_post_data'] = post + except Exception: + meta['request_post_data'] = None + + ctype = (resp_headers.get('content-type') or '').lower() + safe = sanitize(url.replace('https://', '').replace('http://', '')) + h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8] + + body_path = None + excerpt = None + # try textual + try: + if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype: + try: + txt = resp.text() + except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,): + logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '')) + return + # pretty-print JSON if possible + pretty = try_pretty_json(txt) + if pretty is not None: + fname = f"{safe}_{h}_{ts}.json" + body_path = str(output / fname) + save_text(output / fname, pretty) + excerpt = pretty[:800] + else: + fname = f"{safe}_{h}_{ts}.txt" + body_path = str(output / fname) + save_text(output / fname, txt) + excerpt = txt[:800] + else: + try: + data = resp.body() + except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,): + logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '')) + return + ext = guess_ext(ctype) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(output / fname) + save_binary(output / fname, data) + except Exception as e: + try: + # fallback to binary + try: + data = resp.body() + except TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,): + logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '')) + return + ext = guess_ext(ctype) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(output / fname) + save_binary(output / fname, data) + except Exception: + logger.debug('Failed to save body for %s: %s', url, e) + + meta['response_body_file'] = body_path + meta['response_excerpt'] = excerpt + + meta_name = f"{safe}_{h}_{ts}.json" + meta_path = output / meta_name + save_text(meta_path, json.dumps(meta, ensure_ascii=False, indent=2)) + + # append index + try: + with open(index_path, 'a', encoding='utf-8') as idx: + idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(meta_path), 'excerpt': excerpt}, ensure_ascii=False) + '\n') + except Exception: + logger.exception('Failed to write index entry') + + logger.info('Captured %s %s -> %s', rtype, url, meta_path.name) + except Exception as e: + # suppress noisy Playwright shutdown traces + if isinstance(e, TARGET_CLOSED_EXCEPTIONS + (asyncio.CancelledError,)) or \ + (hasattr(e, 'args') and any('Target page, context or browser has been closed' in str(a) for a in e.args)): + logger.debug('Response handler aborted: target closed') + return + logger.exception('Error in response handler') + + # helper to attach listeners to a page + def attach_listeners_to_page(pg): + try: + pg.on('response', on_response) + # log requests and finished requests for debug + def on_request(req): + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n") + except Exception: + pass + + def on_request_finished(req): + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n") + except Exception: + pass + + pg.on('request', on_request) + pg.on('requestfinished', on_request_finished) + # also log request failures for visibility + def on_req_failed(req): + try: + logger.warning('Request failed: %s %s', req.failure, req.url) + except Exception: + pass + pg.on('requestfailed', on_req_failed) + logger.debug('Attached listeners to page: %s', pg.url) + except Exception: + logger.exception('Failed to attach listeners to page') + + try: + # navigation and lifecycle events + def on_navigate(): + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tNAVIGATE\t{pg.url}\n") + except Exception: + pass + pg.on('load', lambda: on_navigate()) + pg.on('domcontentloaded', lambda: on_navigate()) + pg.on('framenavigated', lambda frame: on_navigate()) + except Exception: + pass + + # attach to any existing pages in the context + try: + for pg in context.pages: + attach_listeners_to_page(pg) + except Exception: + logger.debug('No existing pages to attach') + + # attach to future pages + try: + context.on('page', lambda new_page: attach_listeners_to_page(new_page)) + except Exception: + logger.debug('Could not attach context.page listener') + + # also attach at context level to catch responses not tied to a single page + try: + context.on('response', lambda resp: on_response(resp)) + except Exception: + logger.debug('Context-level response listener not available') + + # add context-level request listeners too (some fetches/service-workers + # may appear at the context level rather than page) + try: + def _ctx_request(req): + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n") + except Exception: + pass + def _ctx_req_finished(req): + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tCTX_REQUEST_FINISHED\t{req.method}\t{req.url}\n") + except Exception: + pass + context.on('request', lambda r: _ctx_request(r)) + context.on('requestfinished', lambda r: _ctx_req_finished(r)) + except Exception: + logger.debug('Context-level request listeners not available') + + # try auto-login + if username and password: + attempt_autologin(page, username, password, args.post_login_selector) + + # navigate after listeners attached so we capture initial load + try: + logger.info('Opening %s', args.url) + page.goto(args.url) + except Exception: + logger.debug('Initial navigation failed or was skipped') + + # If interactive and not no-prompt, let user finish manual login/navigation + if not args.no_prompt: + try: + input('When you have logged in and are ready, press Enter to start passive capture...\n') + except Exception: + pass + + logger.info(f'Starting passive response capture (types: {", ".join(sorted(capture_types))})') + + # attach the response handler to the initial page + page.on('response', on_response) + + logger.info('Passive listener attached. Continue browsing in the opened browser window.') + + if args.continuous: + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + logger.info('Interrupted by user; finishing and closing browser') + else: + # one-shot: wait a short while to gather responses then exit + time.sleep(5) + + try: + context.close() + except Exception: + pass + + +if __name__ == '__main__': + main() diff --git a/scrapers/capture_passive_playwright_async.py b/scrapers/capture_passive_playwright_async.py new file mode 100644 index 0000000..67a34a3 --- /dev/null +++ b/scrapers/capture_passive_playwright_async.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +"""Async passive XHR/fetch capture using Playwright's async API. + +This script mirrors the sync version but uses async/await to avoid blocking +the event loop. It attaches listeners before navigating and prints a small +heartbeat that shows how many responses we've captured so you can see +activity live. + +Usage: + python scrapers/capture_passive_playwright_async.py --output-dir xhr_captured_async --continuous +""" +import argparse +import asyncio +import json +import mimetypes +import hashlib +import os +from datetime import datetime, timezone +from pathlib import Path + +from dotenv import load_dotenv +load_dotenv() + +try: + from playwright.async_api import async_playwright +except Exception: + raise + + +def sanitize(s: str) -> str: + out = [] + for ch in s: + if ch.isalnum() or ch in ('-', '_', '.'): + out.append(ch) + else: + out.append('_') + name = ''.join(out) + if len(name) > 180: + name = name[:120] + '_' + hashlib.sha1(s.encode('utf-8')).hexdigest()[:8] + return name + + +def guess_ext(ctype: str) -> str: + if not ctype: + return 'bin' + mt = ctype.split(';')[0].strip() + ext = mimetypes.guess_extension(mt) + if ext: + return ext.lstrip('.') + if mt.startswith('image/'): + return mt.split('/')[1] + if mt == 'application/json': + return 'json' + if mt == 'text/html': + return 'html' + if mt.startswith('text/'): + return 'txt' + return 'bin' + + +def now_ts(): + return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + + +def save_text(path: Path, text: str): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + + +def save_binary(path: Path, data: bytes): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'wb') as f: + f.write(data) + + +async def run(args): + output = Path(args.output_dir) + output.mkdir(parents=True, exist_ok=True) + index_path = output / 'capture_index.jsonl' + + capture_count = 0 + capture_count_lock = asyncio.Lock() + + async def heartbeat(): + while True: + await asyncio.sleep(5) + async with capture_count_lock: + print(f"[heartbeat] captured={capture_count}") + + async with async_playwright() as p: + browser_type = p.chromium + # Launch persistent context + if args.channel: + try: + context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel) + except Exception: + print(f'Launch with channel {args.channel} failed; falling back to bundled Chromium') + context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) + else: + context = await browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) + + page = await context.new_page() + + async def on_response(resp): + nonlocal capture_count + try: + req = resp.request + rtype = (req.resource_type or '').lower() + + # write quick debug line + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tRESP\t{resp.status}\t{rtype}\t{req.url}\n") + except Exception: + pass + + allow = False + if not args.capture_types: + allow = True + elif rtype in args.capture_types: + allow = True + if '/document/content/' in req.url or '/document/summary/' in req.url: + allow = True + if not allow: + return + + url = req.url + ts = now_ts() + resp_headers = dict(resp.headers) + ctype = (resp_headers.get('content-type') or '').lower() + + safe = sanitize(url.replace('https://', '').replace('http://', '')) + h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8] + + body_path = None + excerpt = None + + try: + if 'application/json' in ctype or ctype.startswith('text/') or 'javascript' in ctype: + try: + txt = await resp.text() + except asyncio.CancelledError: + return + except Exception as e: + if 'Target page, context or browser has been closed' in str(e): + return + raise + # try pretty JSON + try: + parsed = json.loads(txt) + pretty = json.dumps(parsed, ensure_ascii=False, indent=2) + fname = f"{safe}_{h}_{ts}.json" + body_path = str(output / fname) + save_text(output / fname, pretty) + excerpt = pretty[:800] + except Exception: + fname = f"{safe}_{h}_{ts}.txt" + body_path = str(output / fname) + save_text(output / fname, txt) + excerpt = txt[:800] + else: + try: + data = await resp.body() + except asyncio.CancelledError: + return + except Exception as e: + if 'Target page, context or browser has been closed' in str(e): + return + raise + ext = guess_ext(ctype) + # Put images into an images/ subdirectory, other binaries into assets/ + if ctype.startswith('image/'): + subdir = output / 'images' + subdir.mkdir(parents=True, exist_ok=True) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(subdir / fname) + save_binary(subdir / fname, data) + else: + subdir = output / 'assets' + subdir.mkdir(parents=True, exist_ok=True) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(subdir / fname) + save_binary(subdir / fname, data) + except Exception as e: + # final fallback: try binary + try: + data = await resp.body() + ext = guess_ext(ctype) + if ctype.startswith('image/'): + subdir = output / 'images' + subdir.mkdir(parents=True, exist_ok=True) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(subdir / fname) + save_binary(subdir / fname, data) + else: + subdir = output / 'assets' + subdir.mkdir(parents=True, exist_ok=True) + fname = f"{safe}_{h}_{ts}.{ext}" + body_path = str(subdir / fname) + save_binary(subdir / fname, data) + except Exception: + # skip if still failing + return + + meta = { + 'url': url, + 'resource_type': rtype, + 'status': resp.status, + 'timestamp': ts, + 'response_headers': resp_headers, + 'response_body_file': body_path, + 'response_excerpt': excerpt, + } + meta_name = f"{safe}_{h}_{ts}.meta.json" + save_text(output / meta_name, json.dumps(meta, ensure_ascii=False, indent=2)) + try: + with open(index_path, 'a', encoding='utf-8') as idx: + idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt}, ensure_ascii=False) + '\n') + except Exception: + pass + + async with capture_count_lock: + capture_count += 1 + + except Exception: + # swallow unexpected exceptions from handler so event loop stays healthy + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tRESP_HANDLER_ERROR\n") + except Exception: + pass + + # helpers to attach to page + def attach_page_listeners(pg): + pg.on('response', lambda r: asyncio.create_task(on_response(r))) + pg.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST\t{req.method}\t{req.url}\n")) + pg.on('requestfinished', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tREQUEST_FINISHED\t{req.method}\t{req.url}\n")) + + # attach listeners to existing pages + for pg in context.pages: + attach_page_listeners(pg) + + # attach to future pages + context.on('page', lambda new_page: attach_page_listeners(new_page)) + + # context-level response and request listeners + context.on('response', lambda r: asyncio.create_task(on_response(r))) + context.on('request', lambda req: open(output / '_debug_events.log', 'a', encoding='utf-8').write(f"{now_ts()}\tCTX_REQUEST\t{req.method}\t{req.url}\n")) + + # Start heartbeat + hb_task = asyncio.create_task(heartbeat()) + + # Navigate after listeners attached + try: + await page.goto(args.url) + except Exception: + pass + + # If interactive, let user log in; otherwise start capture immediately + if not args.no_prompt: + print('When you have logged in in the opened browser, press Enter here to continue and capture...') + try: + await asyncio.get_event_loop().run_in_executor(None, input) + except Exception: + pass + + print(f'Starting async passive capture (types: {args.capture_types})') + + try: + if args.continuous: + # keep running until KeyboardInterrupt + while True: + await asyncio.sleep(1) + else: + await asyncio.sleep(5) + except KeyboardInterrupt: + print('Interrupted; closing...') + finally: + hb_task.cancel() + try: + await context.close() + except Exception: + pass + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--url', default='https://app.statdx.com/') + parser.add_argument('--profile', default='playwright_profile') + parser.add_argument('--output-dir', default='xhr_captured_async') + parser.add_argument('--headless', action='store_true') + parser.add_argument('--continuous', action='store_true') + parser.add_argument('--no-prompt', action='store_true') + parser.add_argument('--channel', default=os.getenv('PLAYWRIGHT_CHROME_CHANNEL', 'chrome')) + parser.add_argument('--capture-types', default='xhr,fetch,document,other') + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + # normalize capture types to a set + args.capture_types = {t.strip().lower() for t in args.capture_types.split(',') if t.strip()} + try: + asyncio.run(run(args)) + except KeyboardInterrupt: + pass diff --git a/scrapers/capture_with_cdp.py b/scrapers/capture_with_cdp.py index 3a4898d..ff51b89 100644 --- a/scrapers/capture_with_cdp.py +++ b/scrapers/capture_with_cdp.py @@ -88,6 +88,8 @@ def main(): parser.add_argument("--headless", action="store_true", help="Run headless") parser.add_argument("--capture-wait", type=int, default=8, help="Seconds to wait after reload to capture XHRs") parser.add_argument("--keep-open", action="store_true", help="Keep browser open after capture") + parser.add_argument("--continuous", action="store_true", help="Continue capturing requests as you browse until interrupted") + parser.add_argument("--no-prompt", action="store_true", help="Do not prompt before starting capture (useful for automated runs)") parser.add_argument("--username", help="STATdx username (or use STATDX_USERNAME env var)") parser.add_argument("--password", help="STATdx password (or use STATDX_PASSWORD env var)") parser.add_argument("--post-login-selector", default="#ds-app", help="Selector to wait for after login") @@ -101,23 +103,31 @@ def main(): with sync_playwright() as p: # try to use installed Chrome first try: - browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome") - page = browser.pages[0] if browser.pages else browser.new_page() + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel="chrome") + page = context.pages[0] if context.pages else context.new_page() except Exception as e: logger.warning("Failed to launch Chrome channel: {}. Falling back to chromium bundle.", e) - browser = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) - page = browser.pages[0] if browser.pages else browser.new_page() + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) + page = context.pages[0] if context.pages else context.new_page() # attach CDP session to the page try: - session = browser.new_cdp_session(page) + session = context.new_cdp_session(page) except Exception as e: logger.error("Failed to create CDP session: {}", e) - browser.close() + try: + context.close() + except Exception: + pass return logger.info("Enabling Network domain on CDP") session.send("Network.enable") + # Disable browser cache to force network requests (helps capture cached items) + try: + session.send("Network.setCacheDisabled", {"cacheDisabled": True}) + except Exception: + logger.debug("Could not disable cache via CDP; continuing without cache disable") seen = set() @@ -137,7 +147,9 @@ def main(): return seen.add(key) - # attempt to get body + # attempt to get body; sometimes responseReceived doesn't allow + # getResponseBody immediately, so we'll try but also rely on + # a loadingFinished handler below. body = None base64_encoded = False try: @@ -146,7 +158,7 @@ def main(): body = result.get("body") base64_encoded = bool(result.get("base64Encoded")) except Exception as e: - logger.debug("Network.getResponseBody failed for {}: {}", url, e) + logger.debug("Network.getResponseBody failed (responseReceived) for {}: {}", url, e) # fallback: if no body, skip writing but still store metadata # build filenames @@ -196,7 +208,54 @@ def main(): # subscribe to CDP event session.on("Network.responseReceived", on_response_received) - logger.info("Opening page: {}. Please log in if needed, then press ENTER to start capture.", args.url) + # loadingFinished is a good fallback: some responses only have bodies + # available after loadingFinished. We'll attempt to fetch body there + # if not previously saved for the same requestId. + def on_loading_finished(params): + try: + requestId = params.get("requestId") + # call the same body retrieval logic via a small wrapper + # create a fake params dict to reuse code + fake = {"requestId": requestId, "response": {}} + # attempt to get the body now + try: + result = session.send("Network.getResponseBody", {"requestId": requestId}) + except Exception: + result = None + if result: + # build a minimal params for saving using response info from CDP + resp = result + # We don't have the URL here; try to fetch it via CDP - getRequestPostData isn't available. + # Instead, reuse on_response_received's behavior by calling getResponseBody and writing + # files directly here when possible. + try: + body = result.get("body") + base64_encoded = bool(result.get("base64Encoded")) + # fetch additional response info via Network.getResponseBody? we already have body. + # write a minimal metadata file + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + safe = sanitize_fn(requestId) + h = hashlib.sha1((requestId + str(time.time())).encode("utf-8")).hexdigest()[:8] + ext = "bin" + try: + data = base64.b64decode(body) if base64_encoded else body.encode("utf-8") + body_filename = f"{safe}_{h}_{ts}.{ext}" + write_file(out_dir / body_filename, data) + meta = {"requestId": requestId, "timestamp": ts, "response_body_file": str(out_dir / body_filename)} + meta_filename = f"{safe}_{h}_{ts}.json" + with open(out_dir / meta_filename, "w", encoding="utf-8") as mf: + json.dump(meta, mf, ensure_ascii=False, indent=2) + logger.info("Saved body via loadingFinished: {}", body_filename) + except Exception: + pass + except Exception: + pass + except Exception: + logger.exception("Error in loadingFinished handler") + + session.on("Network.loadingFinished", on_loading_finished) + + logger.info("Opening page: {}.", args.url) page.goto(args.url) # Automatic login if credentials provided @@ -256,28 +315,46 @@ def main(): except Exception: logger.exception('Automatic login attempt failed') - input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n") + # Start capture: either prompt the user or begin immediately depending on flags + try: + if not args.no_prompt and not args.continuous: + try: + input("After logging in and navigating to the page you want to capture, press Enter to begin capture...\n") + except Exception: + # non-interactive; proceed + pass - logger.info("Reloading page to trigger XHRs and network activity") - page.reload() - - # wait a bit to let XHRs fire and be processed by CDP - wait = args.capture_wait - logger.info("Waiting {} seconds to collect responses...", wait) - time.sleep(wait) - - logger.info("Capture pass complete. Metadata and bodies written to {}", out_dir) - - if args.keep_open: - logger.info("Keeping browser open. Close manually when done.") + logger.info("Reloading page to trigger XHRs and network activity") try: - while True: - time.sleep(1) - except KeyboardInterrupt: - logger.info("Received interrupt, closing.") - browser.close() - else: - browser.close() + page.reload() + except Exception: + logger.exception('Failed to reload page; continuing capture') + + # wait a bit to let XHRs fire and be processed by CDP + wait = args.capture_wait + logger.info("Waiting {} seconds to collect initial responses...", wait) + time.sleep(wait) + + logger.info("Initial capture pass complete. Metadata and bodies written to {}", out_dir) + + if args.continuous: + logger.info("Continuous capture enabled: saving responses as you browse. Press Ctrl+C to stop.") + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + logger.info("Interrupted by user; closing browser and exiting") + else: + logger.info("One-shot capture complete.") + + finally: + try: + context.close() + except Exception: + try: + context.close() + except Exception: + pass if __name__ == "__main__": diff --git a/scrapers/save_page_snapshots.py b/scrapers/save_page_snapshots.py new file mode 100644 index 0000000..1431957 --- /dev/null +++ b/scrapers/save_page_snapshots.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +""" +Save full page HTML snapshots after page loads and periodically while you browse. + +This script launches a persistent Playwright profile (so you can log in), +optionally attempts automatic login, then either performs a one-shot snapshot +or continues saving snapshots as you browse. Snapshots are de-duplicated by +content hash to avoid writing identical files repeatedly. + +Usage examples: + # manual login, one-shot snapshot after reload + python scrapers/save_page_snapshots.py --url https://app.statdx.com/ --output-dir xhr_captured + + # auto-login and continuous snapshots while you browse + python scrapers/save_page_snapshots.py --continuous --username you --password secret --output-dir xhr_captured + +""" +from pathlib import Path +import argparse +import time +import hashlib +import json +import mimetypes +import os +from datetime import datetime, timezone + +try: + from loguru import logger +except Exception: + import logging + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + logger = logging.getLogger("save_page_snapshots") + +from playwright.sync_api import sync_playwright +from dotenv import load_dotenv + +# load .env +load_dotenv() + + +def sanitize_filename(s: str) -> str: + safe = [] + for ch in s: + if ch.isalnum() or ch in ('-', '_', '.'): + safe.append(ch) + else: + safe.append('-') + name = ''.join(safe) + if len(name) > 180: + h = hashlib.sha1(s.encode('utf-8')).hexdigest()[:8] + name = name[:120] + '-' + h + return name + + +def write_text(path: Path, text: str): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + f.write(text) + + +def write_json(path: Path, obj): + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, ensure_ascii=False, indent=2) + + +def now_ts(): + return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ') + + +def snapshot_page(page, out_dir: Path, index_path: Path, last_hashes: dict, min_interval: float = 3.0): + """Capture page content and write HTML + meta if content changed or enough time passed.""" + try: + url = page.url + except Exception: + url = 'unknown' + + try: + content = page.content() + except Exception as e: + logger.exception('Failed to get page.content(): {}', e) + return None + + h = hashlib.sha1(content.encode('utf-8')).hexdigest() + prev = last_hashes.get(url) + ts = now_ts() + + # If unchanged and we saved recently, skip + if prev and prev.get('hash') == h and (time.time() - prev.get('ts', 0)) < min_interval: + logger.debug('Snapshot skipped (no change) for {}', url) + return None + + # prepare filenames + name = sanitize_filename(url.replace('https://', '').replace('http://', '')) + short = h[:8] + html_name = f"{name}_{short}_{ts}.html" + meta_name = f"{name}_{short}_{ts}.json" + + html_path = out_dir / html_name + meta_path = out_dir / meta_name + + try: + write_text(html_path, content) + except Exception as e: + logger.exception('Failed to write HTML snapshot: {}', e) + return None + + meta = { + 'url': url, + 'timestamp': ts, + 'file': str(html_path), + 'sha1': h, + 'title': (page.title() if page else None), + } + try: + write_json(meta_path, meta) + except Exception: + logger.exception('Failed to write meta file') + + # append to index + try: + entry = {'url': url, 'timestamp': ts, 'html_file': str(html_path), 'meta_file': str(meta_path), 'sha1': h} + with open(index_path, 'a', encoding='utf-8') as idx: + idx.write(json.dumps(entry, ensure_ascii=False) + '\n') + except Exception: + logger.exception('Failed to append to index') + + last_hashes[url] = {'hash': h, 'ts': time.time()} + logger.info('Saved snapshot: {} ({} bytes)', html_path.name, len(content)) + return html_path + + +def attempt_autologin(page, username: str, password: str, post_login_selector: str, wait_after: float = 3.0): + if not username or not password: + return False + logger.info('Attempting automatic login') + # Common selectors for STATdx and similar pages + user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]'] + pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector'] + + user_sel = None + for sel in user_selectors: + try: + if page.query_selector(sel): + user_sel = sel + break + except Exception: + continue + + pass_sel = None + for sel in pass_selectors: + try: + if page.query_selector(sel): + pass_sel = sel + break + except Exception: + continue + + if not user_sel or not pass_sel: + logger.warning('Could not find username/password fields for auto-login') + return False + + try: + page.fill(user_sel, username) + time.sleep(0.1) + page.fill(pass_sel, password) + time.sleep(0.1) + # try submit + submit_selectors = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")'] + clicked = False + for s in submit_selectors: + try: + btn = page.query_selector(s) + if btn: + btn.click() + clicked = True + logger.info('Clicked submit button using selector: {}', s) + break + except Exception: + continue + if not clicked: + page.press(pass_sel, 'Enter') + + # wait for post-login selector + try: + if post_login_selector: + page.wait_for_selector(post_login_selector, timeout=10000) + except Exception: + logger.info('Post-login selector not found within timeout') + time.sleep(wait_after) + return True + except Exception: + logger.exception('Auto-login failed') + return False + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--url', '-u', default='https://app.statdx.com/') + parser.add_argument('--profile', '-p', default='playwright_profile') + parser.add_argument('--output-dir', '-o', default='xhr_captured') + parser.add_argument('--headless', action='store_true') + parser.add_argument('--continuous', action='store_true', help='Keep capturing while you browse until interrupted') + parser.add_argument('--capture-wait', type=float, default=6.0, help='Seconds to wait before initial snapshot') + parser.add_argument('--poll-interval', type=float, default=5.0, help='Periodic snapshot interval when continuous') + parser.add_argument('--min-interval', type=float, default=3.0, help='Minimum seconds between snapshots for same URL') + parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)') + parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)') + parser.add_argument('--post-login-selector', default='#ds-app') + parser.add_argument('--no-prompt', action='store_true') + args = parser.parse_args() + + out_dir = Path(args.output_dir) + out_dir.mkdir(parents=True, exist_ok=True) + index_path = out_dir / 'snapshot_index.jsonl' + + username = args.username or os.getenv('STATDX_USERNAME') + password = args.password or os.getenv('STATDX_PASSWORD') + + last_hashes = {} + + with sync_playwright() as p: + try: + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel='chrome') + except Exception: + context = p.chromium.launch_persistent_context(user_data_dir=args.profile, headless=args.headless) + + page = context.new_page() + logger.info('Opening %s', args.url) + page.goto(args.url) + + # attempt auto-login + if username and password: + attempt_autologin(page, username, password, args.post_login_selector) + + # wait before initial snapshot to let in-flight XHRs populate DOM + logger.info('Waiting %.1f sec before initial snapshot', args.capture_wait) + time.sleep(args.capture_wait) + + # initial snapshot + snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval) + + if args.continuous: + logger.info('Continuous snapshot mode: saving on navigation and every %.1f sec', args.poll_interval) + + # on navigation events, take a snapshot (debounced by min_interval) + def on_load(): + try: + snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval) + except Exception: + logger.exception('Error during on_load snapshot') + + page.on('load', lambda: on_load()) + page.on('domcontentloaded', lambda: on_load()) + page.on('framenavigated', lambda frame: on_load()) + + try: + # periodic polling snapshot + while True: + time.sleep(args.poll_interval) + snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval) + except KeyboardInterrupt: + logger.info('Interrupted by user, closing') + else: + # one-shot: save a final snapshot after a short delay to capture dynamic updates + try: + time.sleep(1.0) + snapshot_page(page, out_dir, index_path, last_hashes, min_interval=args.min_interval) + except Exception: + logger.exception('Error taking final snapshot') + + try: + context.close() + except Exception: + pass + + +if __name__ == '__main__': + main() diff --git a/scrapers/statdx_live_capture.py b/scrapers/statdx_live_capture.py index 14446b5..07ac14d 100644 --- a/scrapers/statdx_live_capture.py +++ b/scrapers/statdx_live_capture.py @@ -22,7 +22,6 @@ Notes / limitations: import os import argparse -import logging import time import threading import sys @@ -32,9 +31,12 @@ from urllib.parse import urlparse from dotenv import load_dotenv import mimetypes +from loguru import logger + load_dotenv() -logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger.remove() +logger.add(sys.stderr, level="INFO") def sanitize_filename(url: str) -> str: @@ -79,7 +81,7 @@ def auto_scroll(page, step_ms: int = 50, step_px: int = 300): step_ms, ) except Exception: - logging.exception('auto_scroll failed') + logger.exception('auto_scroll failed') def save_response(resp, out_dir: str): @@ -112,7 +114,7 @@ def save_response(resp, out_dir: str): filename_base = sanitize_filename(url) out_path = os.path.join(out_dir, filename_base + ext) - logging.info('Saving %s -> %s (content-type: %s)', url, out_path, ctype.split(';')[0]) + logger.info(f'Saving {url} -> {out_path} (content-type: {ctype.split(";")[0]})') if save_as_text: try: @@ -136,7 +138,7 @@ def save_response(resp, out_dir: str): return True except Exception as e: - logging.exception('Failed to save response: %s', e) + logger.exception(f'Failed to save response: {e}') return False @@ -147,21 +149,21 @@ def save_page_snapshot(page, out_dir: str, reason: str = ''): filename_base = sanitize_filename(url) safe_reason = ''.join(c if c.isalnum() or c in ('-', '_') else '_' for c in reason)[:40] out_path = os.path.join(out_dir, f"snapshot_{filename_base}_{ts}_{safe_reason}.html") - logging.info('Saving page snapshot: %s -> %s', url, out_path) + logger.info(f'Saving page snapshot: {url} -> {out_path}') content = page.content() with open(out_path, 'w', encoding='utf-8') as fh: fh.write(content) # Screenshots are disabled (not saving visual snapshots per user request) return True except Exception: - logging.exception('Failed to save page snapshot') + logger.exception('Failed to save page snapshot') return False def main(): parser = argparse.ArgumentParser(description='Live-capture browsing responses using Playwright') parser.add_argument('--output', '-o', default='captured', help='Directory to save captured responses') - parser.add_argument('--profile', '-p', default='.playwright_profile', help='Persistent profile directory') + parser.add_argument('--profile', '-p', default='playwright_profile', help='Persistent profile directory') parser.add_argument('--domain', '-d', default=os.getenv('STATDX_DOMAIN', 'app.statdx.com'), help='Only capture responses whose URL contains this domain') parser.add_argument('--headless', action='store_true', help='Run headless (no visible browser)') parser.add_argument('--snapshot-delay', type=float, default=0.5, help='Seconds to wait after an XHR before saving a snapshot') @@ -182,7 +184,7 @@ def main(): with sync_playwright() as p: # Use persistent context so cookies/localStorage persist between runs browser_type = p.chromium - logging.info('Launching Chromium (headless=%s) with profile: %s, channel=%s', args.headless, args.profile, args.channel) + logger.info(f'Launching Chromium (headless={args.headless}) with profile: {args.profile}, channel={args.channel}') if args.channel: context = browser_type.launch_persistent_context(user_data_dir=args.profile, headless=args.headless, channel=args.channel) else: @@ -198,6 +200,7 @@ def main(): pass def handle_response(resp): + logger.debug(f'RESPONSE: {resp.url} ({resp.request.resource_type})') try: if args.domain not in resp.url: return @@ -214,12 +217,12 @@ def main(): try: auto_scroll(page) except Exception: - logging.exception('auto-scroll failed') + logger.exception('auto-scroll failed') save_page_snapshot(page, args.output, reason=rtype) except Exception: - logging.exception('snapshot after response failed') + logger.exception('snapshot after response failed') except Exception: - logging.exception('Error in response handler') + logger.exception('Error in response handler') page.on('response', handle_response) @@ -227,19 +230,19 @@ def main(): # Log console messages and page errors to help diagnose missing content def on_console(msg): try: - logging.info('PAGE LOG [%s] %s', msg.type, msg.text) + logger.info(f'PAGE LOG [{msg.type}] {msg.text}') except Exception: - logging.info('PAGE LOG: %s', msg) + logger.info(f'PAGE LOG: {msg}') def on_page_error(exc): - logging.error('PAGE ERROR: %s', exc) + logger.error(f'PAGE ERROR: {exc}') page.on('console', on_console) page.on('pageerror', on_page_error) - logging.info('Open the browser window and interact with the site. Captured responses will be saved to: %s', args.output) - logging.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).') - logging.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.') + logger.info(f'Open the browser window and interact with the site. Captured responses will be saved to: {args.output}') + logger.info('Press Ctrl+C here in the terminal to exit and close the browser (profile saved).') + logger.info('Type "s" and press Enter (or just press Enter) in this terminal to save a manual snapshot of the current page.') # Navigate to the domain root so you can log in manually if needed start_url = f'https://{args.domain}/' @@ -247,27 +250,27 @@ def main(): # Optionally wait for a selector that indicates the page finished rendering if args.wait_for_selector: try: - logging.info('Waiting for selector: %s', args.wait_for_selector) + logger.info(f'Waiting for selector: {args.wait_for_selector}') page.wait_for_selector(args.wait_for_selector, timeout=20000) except Exception: - logging.exception('Waiting for selector timed out') + logger.exception('Waiting for selector timed out') def stdin_watcher(): # Run in a background thread so the main thread can keep the browser running - logging.debug('stdin watcher started') + logger.debug('stdin watcher started') for line in sys.stdin: cmd = line.strip().lower() if cmd == '' or cmd == 's' or cmd == 'snapshot' or cmd == 'save': - logging.info('Manual snapshot requested via stdin') + logger.info('Manual snapshot requested via stdin') try: if args.auto_scroll: try: auto_scroll(page) except Exception: - logging.exception('manual auto-scroll failed') + logger.exception('manual auto-scroll failed') save_page_snapshot(page, args.output, reason='manual') except Exception: - logging.exception('manual snapshot failed') + logger.exception('manual snapshot failed') watcher = threading.Thread(target=stdin_watcher, daemon=True) watcher.start() @@ -277,7 +280,7 @@ def main(): while True: time.sleep(1) except KeyboardInterrupt: - logging.info('Shutting down...') + logger.info('Shutting down...') finally: context.close() diff --git a/xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json b/xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json deleted file mode 100644 index 4b27bd7..0000000 --- a/xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "url": "https://app.statdx.com/csrf/token", - "method": "GET", - "resource_type": "xhr", - "status": 200, - "request_headers": { - "sec-ch-ua-platform": "\"Linux\"", - "referer": "https://app.statdx.com/", - "x-requested-with": "XMLHttpRequest", - "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36", - "accept": "*/*", - "sec-ch-ua": "\"Not=A?Brand\";v=\"24\", \"Chromium\";v=\"140\"", - "sec-ch-ua-mobile": "?0" - }, - "response_headers": { - "version-pss-dsjs-config": "2025.9.11-134022-sdx-ASTRO-e2575728", - "version-pss-dsjs": "2025.9.11-134022-sdx-ASTRO-e2575728", - "content-encoding": "gzip", - "cf-cache-status": "DYNAMIC", - "etag": "W/\"10c-vLpriK5k7wHbwVGauL6RBVcuBHQ\"", - "x-ua-compatible": "IE=edge", - "date": "Tue, 14 Oct 2025 12:41:21 GMT", - "content-type": "application/json; charset=utf-8", - "vary": "Accept-Encoding", - "version-pss-sparks-event-client": "2.1.1", - "strict-transport-security": "max-age=31536000; includeSubDomains", - "content-security-policy": "script-src 'self' 'unsafe-eval' 'unsafe-inline' js-agent.newrelic.com static.cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; style-src 'self' 'unsafe-inline' fonts.googleapis.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; img-src 'self' data: app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com *.cloudfront.net; media-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com *.cloudfront.net; connect-src 'self' ws: cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; frame-src 'self' app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com; worker-src 'self' blob: scopus.com service.elsevier.com pendo.reaxys.com; object-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com", - "cache-control": "no-cache, no-store", - "tdm-reservation": "1", - "x-ratelimit-reset": "1760445742", - "hostname": "sdxapp3", - "cf-ray": "98e726878fefbea1-LHR", - "x-ratelimit-remaining": "499", - "version-pss-cmejs": "9.3.3", - "tdm-policy": "https://www.elsevier.com/tdm/tdmrep-policy.json", - "x-ratelimit-limit": "500", - "version-pss-authenticationjs": "7.1.0", - "x-powered-by": "Express", - "server": "cloudflare" - }, - "timestamp": "20251014T124122Z", - "request_post_data": null, - "response_text": "{\"token\":\"c67a4a30b300096e11483fc08f9dcb69e679ef1152816abe6bcad85c694959d5ba2e29cab29da9e13a743b29daac792a44a0af3bcf7f0b6476c1691d52b01918472529e208a3431dc22a435435c7947b1212f546ae3b618ad1701e1f37f616c6f6a76cfa179fb82706566692759ca11b617ba0d9501d9ee88a7e1e6ac2c8be39\"}", - "response_body_file": null -} \ No newline at end of file