.

scrapers/capture_passive_playwright_async.py

@@ -62,19 +62,30 @@ def now_ts():
     return datetime.now(timezone.utc).strftime('%Y%m%dT%H%M%SZ')
 
 
-def save_text(path: Path, text: str):
-    path.parent.mkdir(parents=True, exist_ok=True)
-    with open(path, 'w', encoding='utf-8') as f:
-        f.write(text)
-
-
 def save_binary(path: Path, data: bytes):
     path.parent.mkdir(parents=True, exist_ok=True)
     with open(path, 'wb') as f:
         f.write(data)
 
 
+def save_text(path: Path, text: str):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with open(path, 'w', encoding='utf-8') as f:
+        f.write(text)
+
+
+def try_pretty_json(s: str):
+    try:
+        parsed = json.loads(s)
+        return json.dumps(parsed, ensure_ascii=False, indent=2)
+    except Exception:
+        return None
+
+
 async def run(args):
+    in_memory_index = {}
+
+    # runtime/shared state for handlers
     output = Path(args.output_dir)
     output.mkdir(parents=True, exist_ok=True)
     index_path = output / 'capture_index.jsonl'
@@ -84,33 +95,11 @@ async def run(args):
 
     async def heartbeat():
         while True:
+            try:
+                print(f"{now_ts()}\tHEARTBEAT\tcaptured={capture_count}")
+            except Exception:
+                pass
             await asyncio.sleep(5)
-            async with capture_count_lock:
-                print(f"[heartbeat] captured={capture_count}")
-        
-    # Build an in-memory index of content_hash -> list of index entries for quick dedupe lookups.
-    # If older index lines lack a content_hash but reference a body file, hash that file once at startup.
-    in_memory_index = {}
-    try:
-        if index_path.exists():
-            with open(index_path, 'r', encoding='utf-8') as idxf:
-                for line in idxf:
-                    try:
-                        j = json.loads(line)
-                    except Exception:
-                        continue
-                    ch = j.get('content_hash')
-                    body = j.get('body_file')
-                    if not ch and body and os.path.exists(body):
-                        try:
-                            with open(body, 'rb') as bf:
-                                ch = hashlib.sha256(bf.read()).hexdigest()
-                        except Exception:
-                            ch = None
-                    if ch:
-                        in_memory_index.setdefault(ch, []).append(j)
-    except Exception:
-        in_memory_index = {}
 
     async with async_playwright() as p:
         browser_type = p.chromium