diff --git a/scrapers/capture_passive_playwright_async.py b/scrapers/capture_passive_playwright_async.py index 67a34a3..680fdfa 100644 --- a/scrapers/capture_passive_playwright_async.py +++ b/scrapers/capture_passive_playwright_async.py @@ -115,21 +115,31 @@ async def run(args): except Exception: pass + # determine response content-type early so we can allow images + resp_headers = dict(resp.headers) + ctype = (resp_headers.get('content-type') or '').lower() + allow = False if not args.capture_types: allow = True elif rtype in args.capture_types: allow = True + # Heuristic: always capture known document endpoints if '/document/content/' in req.url or '/document/summary/' in req.url: allow = True + # Always capture images regardless of capture_types + if ctype.startswith('image/'): + allow = True if not allow: + try: + with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg: + dbg.write(f"{now_ts()}\tSKIP\t{rtype}\t{req.url}\t{ctype}\n") + except Exception: + pass return url = req.url ts = now_ts() - resp_headers = dict(resp.headers) - ctype = (resp_headers.get('content-type') or '').lower() - safe = sanitize(url.replace('https://', '').replace('http://', '')) h = hashlib.sha1((url + ts).encode('utf-8')).hexdigest()[:8] @@ -182,7 +192,7 @@ async def run(args): fname = f"{safe}_{h}_{ts}.{ext}" body_path = str(subdir / fname) save_binary(subdir / fname, data) - except Exception as e: + except Exception: # final fallback: try binary try: data = await resp.body() @@ -222,7 +232,6 @@ async def run(args): async with capture_count_lock: capture_count += 1 - except Exception: # swallow unexpected exceptions from handler so event loop stays healthy try: