Ross
491fca2f42
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
45 lines
5.6 KiB
JSON
45 lines
5.6 KiB
JSON
{
|
|
"url": "https://app.statdx.com/csrf/token",
|
|
"method": "GET",
|
|
"resource_type": "xhr",
|
|
"status": 200,
|
|
"request_headers": {
|
|
"sec-ch-ua-platform": "\"Linux\"",
|
|
"referer": "https://app.statdx.com/",
|
|
"x-requested-with": "XMLHttpRequest",
|
|
"user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
|
|
"accept": "*/*",
|
|
"sec-ch-ua": "\"Not=A?Brand\";v=\"24\", \"Chromium\";v=\"140\"",
|
|
"sec-ch-ua-mobile": "?0"
|
|
},
|
|
"response_headers": {
|
|
"version-pss-dsjs-config": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
|
"version-pss-dsjs": "2025.9.11-134022-sdx-ASTRO-e2575728",
|
|
"content-encoding": "gzip",
|
|
"cf-cache-status": "DYNAMIC",
|
|
"etag": "W/\"10c-vLpriK5k7wHbwVGauL6RBVcuBHQ\"",
|
|
"x-ua-compatible": "IE=edge",
|
|
"date": "Tue, 14 Oct 2025 12:41:21 GMT",
|
|
"content-type": "application/json; charset=utf-8",
|
|
"vary": "Accept-Encoding",
|
|
"version-pss-sparks-event-client": "2.1.1",
|
|
"strict-transport-security": "max-age=31536000; includeSubDomains",
|
|
"content-security-policy": "script-src 'self' 'unsafe-eval' 'unsafe-inline' js-agent.newrelic.com static.cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; style-src 'self' 'unsafe-inline' fonts.googleapis.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; img-src 'self' data: app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io cdn.pendo.io cdn.eu.pendo.io us1.cdn.pendo.io cdn.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com *.cloudfront.net; media-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com *.cloudfront.net; connect-src 'self' ws: cloudflareinsights.com app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io data.pendo.io data.eu.pendo.io us1.data.pendo.io data.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com *.nr-data.net pendo-io-static.storage.googleapis.com pendo-eu-static.storage.googleapis.com pendo-us1-static.storage.googleapis.com pendo-jp-prod-static.storage.googleapis.com pendo-static-5551907851993088.storage.googleapis.com pendo-eu-static-5551907851993088.storage.googleapis.com pendo-us1-static-5551907851993088.storage.googleapis.com pendo-jp-prod-static-5551907851993088.storage.googleapis.com pendo-static-5582159194488832.storage.googleapis.com pendo-static-6012908437241856.storage.googleapis.com pendo-static-5095337838772224.storage.googleapis.com pendo-static-6027490506309632.storage.googleapis.com; frame-src 'self' app.pendo.io app.eu.pendo.io us1.app.pendo.io app.jpn.pendo.io scopus.com service.elsevier.com pendo.reaxys.com; worker-src 'self' blob: scopus.com service.elsevier.com pendo.reaxys.com; object-src 'self' data: scopus.com service.elsevier.com pendo.reaxys.com",
|
|
"cache-control": "no-cache, no-store",
|
|
"tdm-reservation": "1",
|
|
"x-ratelimit-reset": "1760445742",
|
|
"hostname": "sdxapp3",
|
|
"cf-ray": "98e726878fefbea1-LHR",
|
|
"x-ratelimit-remaining": "499",
|
|
"version-pss-cmejs": "9.3.3",
|
|
"tdm-policy": "https://www.elsevier.com/tdm/tdmrep-policy.json",
|
|
"x-ratelimit-limit": "500",
|
|
"version-pss-authenticationjs": "7.1.0",
|
|
"x-powered-by": "Express",
|
|
"server": "cloudflare"
|
|
},
|
|
"timestamp": "20251014T124122Z",
|
|
"request_post_data": null,
|
|
"response_text": "{\"token\":\"c67a4a30b300096e11483fc08f9dcb69e679ef1152816abe6bcad85c694959d5ba2e29cab29da9e13a743b29daac792a44a0af3bcf7f0b6476c1691d52b01918472529e208a3431dc22a435435c7947b1212f546ae3b618ad1701e1f37f616c6f6a76cfa179fb82706566692759ca11b617ba0d9501d9ee88a7e1e6ac2c8be39\"}",
|
|
"response_body_file": null
|
|
} |