diff --git a/scrapers/capture_passive_playwright.py b/scrapers/capture_passive_playwright.py index ff32562..55d6f1a 100644 --- a/scrapers/capture_passive_playwright.py +++ b/scrapers/capture_passive_playwright.py @@ -288,7 +288,11 @@ def main(): logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '')) return ext = guess_ext(ctype) - fname = f"{safe}_{h}_{ts}.{ext}" + # For images, avoid per-capture timestamps: use stable name with hash + if ctype.startswith('image/'): + fname = f"{safe}_{h}.{ext}" + else: + fname = f"{safe}_{h}_{ts}.{ext}" body_path = str(output / fname) save_binary(output / fname, data) except Exception as e: @@ -300,7 +304,10 @@ def main(): logger.debug('Response body unavailable (target closed) for %s', getattr(resp, 'url', '')) return ext = guess_ext(ctype) - fname = f"{safe}_{h}_{ts}.{ext}" + if ctype.startswith('image/'): + fname = f"{safe}_{h}.{ext}" + else: + fname = f"{safe}_{h}_{ts}.{ext}" body_path = str(output / fname) save_binary(output / fname, data) except Exception: diff --git a/scrapers/capture_passive_playwright_async.py b/scrapers/capture_passive_playwright_async.py index 5da879a..86e7ed8 100644 --- a/scrapers/capture_passive_playwright_async.py +++ b/scrapers/capture_passive_playwright_async.py @@ -289,9 +289,10 @@ async def run(args): ext = guess_ext(ctype) # Put images into an images/ subdirectory, other binaries into assets/ if ctype.startswith('image/'): + # Images don't need per-capture timestamps; use stable names with hash subdir = output / 'images' subdir.mkdir(parents=True, exist_ok=True) - fname = f"{safe}_{h}_{ts}.{ext}" + fname = f"{safe}_{h}.{ext}" body_path = str(subdir / fname) save_binary(subdir / fname, data) try: @@ -314,9 +315,10 @@ async def run(args): data = await resp.body() ext = guess_ext(ctype) if ctype.startswith('image/'): + # Fallback path for images: same stable naming without timestamp subdir = output / 'images' subdir.mkdir(parents=True, exist_ok=True) - fname = f"{safe}_{h}_{ts}.{ext}" + fname = f"{safe}_{h}.{ext}" body_path = str(subdir / fname) save_binary(subdir / fname, data) try: diff --git a/scrapers/capture_with_cdp.py b/scrapers/capture_with_cdp.py index ff51b89..cfdacf9 100644 --- a/scrapers/capture_with_cdp.py +++ b/scrapers/capture_with_cdp.py @@ -31,7 +31,36 @@ try: except Exception: import logging logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") - logger = logging.getLogger("capture_with_cdp") + # Provide a lightweight wrapper that supports loguru-style "{}" formatting + class _LoggerWrapper: + def __init__(self, name: str): + self._lg = logging.getLogger(name) + + def _fmt(self, msg: str, *args): + try: + if args: + return msg.format(*args) + except Exception: + pass + return msg + + def info(self, msg: str, *args, **kwargs): + self._lg.info(self._fmt(msg, *args), **kwargs) + + def warning(self, msg: str, *args, **kwargs): + self._lg.warning(self._fmt(msg, *args), **kwargs) + + def debug(self, msg: str, *args, **kwargs): + self._lg.debug(self._fmt(msg, *args), **kwargs) + + def exception(self, msg: str = "", *args, **kwargs): + try: + self._lg.exception(self._fmt(msg, *args), **kwargs) + except Exception: + # fallback: log the exception without formatted message + self._lg.exception(msg) + + logger = _LoggerWrapper("capture_with_cdp") # type: ignore from playwright.sync_api import sync_playwright from dotenv import load_dotenv import os @@ -166,7 +195,11 @@ def main(): h = hashlib.sha1((url + str(time.time())).encode("utf-8")).hexdigest()[:8] # determine extension ext = guess_ext_from_mime(mime) - body_filename = f"{safe}_{h}_{ts}.{ext}" + # For images, avoid per-capture timestamps: use stable name with hash + if mime and mime.startswith('image/'): + body_filename = f"{safe}_{h}.{ext}" + else: + body_filename = f"{safe}_{h}_{ts}.{ext}" meta_filename = f"{safe}_{h}_{ts}.json" meta = { @@ -214,9 +247,6 @@ def main(): def on_loading_finished(params): try: requestId = params.get("requestId") - # call the same body retrieval logic via a small wrapper - # create a fake params dict to reuse code - fake = {"requestId": requestId, "response": {}} # attempt to get the body now try: result = session.send("Network.getResponseBody", {"requestId": requestId}) @@ -224,13 +254,15 @@ def main(): result = None if result: # build a minimal params for saving using response info from CDP - resp = result # We don't have the URL here; try to fetch it via CDP - getRequestPostData isn't available. # Instead, reuse on_response_received's behavior by calling getResponseBody and writing # files directly here when possible. try: body = result.get("body") base64_encoded = bool(result.get("base64Encoded")) + if body is None: + # no body to save + return # fetch additional response info via Network.getResponseBody? we already have body. # write a minimal metadata file ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") @@ -239,7 +271,8 @@ def main(): ext = "bin" try: data = base64.b64decode(body) if base64_encoded else body.encode("utf-8") - body_filename = f"{safe}_{h}_{ts}.{ext}" + # requestId-based captures: use stable name without timestamp for binary/image-like items + body_filename = f"{safe}_{h}.{ext}" write_file(out_dir / body_filename, data) meta = {"requestId": requestId, "timestamp": ts, "response_body_file": str(out_dir / body_filename)} meta_filename = f"{safe}_{h}_{ts}.json"