Add Playwright-based scraping and login scripts for STATdx
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
This commit is contained in:
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Scan captured JSON files and extract likely title/name fields.
|
||||
|
||||
Usage:
|
||||
python scrapers/parse_captured_json.py --dir captured --out parsed_topics.json
|
||||
|
||||
This prints a brief summary to stdout and writes a JSON file with discovered records.
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import argparse
|
||||
import logging
|
||||
from glob import glob
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
||||
|
||||
LIKELY_KEYS = ['title', 'name', 'label', 'displayName', 'heading']
|
||||
|
||||
|
||||
def extract_from_obj(obj):
|
||||
found = []
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
if k in LIKELY_KEYS and isinstance(v, (str, int, float)):
|
||||
found.append({'key': k, 'value': str(v)})
|
||||
else:
|
||||
found.extend(extract_from_obj(v))
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
found.extend(extract_from_obj(item))
|
||||
return found
|
||||
|
||||
|
||||
def process_file(path):
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as fh:
|
||||
data = json.load(fh)
|
||||
except Exception:
|
||||
# skip files that aren't valid JSON
|
||||
return []
|
||||
return extract_from_obj(data)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--dir', '-d', default='captured')
|
||||
parser.add_argument('--out', '-o', default='parsed_topics.json')
|
||||
args = parser.parse_args()
|
||||
|
||||
results = {}
|
||||
files = glob(os.path.join(args.dir, '**', '*.json'), recursive=True)
|
||||
logging.info('Scanning %d JSON files under %s', len(files), args.dir)
|
||||
for f in files:
|
||||
items = process_file(f)
|
||||
if items:
|
||||
results[f] = items
|
||||
|
||||
with open(args.out, 'w', encoding='utf-8') as fh:
|
||||
json.dump(results, fh, indent=2)
|
||||
logging.info('Wrote parsed results to %s. %d files had matches.', args.out, len(results))
|
||||
|
||||
# Print a short summary
|
||||
for path, items in results.items():
|
||||
print('\nFile:', path)
|
||||
for it in items[:10]:
|
||||
print(' -', it['key'], ':', it['value'][:120])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user