#!/usr/bin/env python3
"""Scan captured JSON files and extract likely title/name fields.

Usage:
  python scrapers/parse_captured_json.py --dir captured --out parsed_topics.json

This prints a brief summary to stdout and writes a JSON file with discovered records.
"""

import os
import json
import argparse
import logging
from glob import glob

logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')

LIKELY_KEYS = ['title', 'name', 'label', 'displayName', 'heading']


def extract_from_obj(obj):
    found = []
    if isinstance(obj, dict):
        for k, v in obj.items():
            if k in LIKELY_KEYS and isinstance(v, (str, int, float)):
                found.append({'key': k, 'value': str(v)})
            else:
                found.extend(extract_from_obj(v))
    elif isinstance(obj, list):
        for item in obj:
            found.extend(extract_from_obj(item))
    return found


def process_file(path):
    try:
        with open(path, 'r', encoding='utf-8') as fh:
            data = json.load(fh)
    except Exception:
        # skip files that aren't valid JSON
        return []
    return extract_from_obj(data)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--dir', '-d', default='captured')
    parser.add_argument('--out', '-o', default='parsed_topics.json')
    args = parser.parse_args()

    results = {}
    files = glob(os.path.join(args.dir, '**', '*.json'), recursive=True)
    logging.info('Scanning %d JSON files under %s', len(files), args.dir)
    for f in files:
        items = process_file(f)
        if items:
            results[f] = items

    with open(args.out, 'w', encoding='utf-8') as fh:
        json.dump(results, fh, indent=2)
    logging.info('Wrote parsed results to %s. %d files had matches.', args.out, len(results))

    # Print a short summary
    for path, items in results.items():
        print('\nFile:', path)
        for it in items[:10]:
            print(' -', it['key'], ':', it['value'][:120])


if __name__ == '__main__':
    main()