Ross
491fca2f42
- Implemented `statdx_play_chrome.py` to launch Chrome via Playwright, allowing for headful browsing and capturing XHR requests. - Created `statdx_playwright.py` for headless scraping of the STATdx application, including automatic login and HTML saving. - Developed `statdx_requests.py` for form-based login and scraping, providing a fallback for sites using traditional authentication. - Added example HTML output from Playwright scraping to `topics_playwright.html`. - Captured XHR request data in JSON format for analysis in `xhr_captured/app.statdx.com_csrf_token_8000d0ed_20251014T124122Z.json`.
72 lines
2.0 KiB
Python
72 lines
2.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Scan captured JSON files and extract likely title/name fields.
|
|
|
|
Usage:
|
|
python scrapers/parse_captured_json.py --dir captured --out parsed_topics.json
|
|
|
|
This prints a brief summary to stdout and writes a JSON file with discovered records.
|
|
"""
|
|
|
|
import os
|
|
import json
|
|
import argparse
|
|
import logging
|
|
from glob import glob
|
|
|
|
logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
|
|
|
|
LIKELY_KEYS = ['title', 'name', 'label', 'displayName', 'heading']
|
|
|
|
|
|
def extract_from_obj(obj):
|
|
found = []
|
|
if isinstance(obj, dict):
|
|
for k, v in obj.items():
|
|
if k in LIKELY_KEYS and isinstance(v, (str, int, float)):
|
|
found.append({'key': k, 'value': str(v)})
|
|
else:
|
|
found.extend(extract_from_obj(v))
|
|
elif isinstance(obj, list):
|
|
for item in obj:
|
|
found.extend(extract_from_obj(item))
|
|
return found
|
|
|
|
|
|
def process_file(path):
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as fh:
|
|
data = json.load(fh)
|
|
except Exception:
|
|
# skip files that aren't valid JSON
|
|
return []
|
|
return extract_from_obj(data)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--dir', '-d', default='captured')
|
|
parser.add_argument('--out', '-o', default='parsed_topics.json')
|
|
args = parser.parse_args()
|
|
|
|
results = {}
|
|
files = glob(os.path.join(args.dir, '**', '*.json'), recursive=True)
|
|
logging.info('Scanning %d JSON files under %s', len(files), args.dir)
|
|
for f in files:
|
|
items = process_file(f)
|
|
if items:
|
|
results[f] = items
|
|
|
|
with open(args.out, 'w', encoding='utf-8') as fh:
|
|
json.dump(results, fh, indent=2)
|
|
logging.info('Wrote parsed results to %s. %d files had matches.', args.out, len(results))
|
|
|
|
# Print a short summary
|
|
for path, items in results.items():
|
|
print('\nFile:', path)
|
|
for it in items[:10]:
|
|
print(' -', it['key'], ':', it['value'][:120])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|