feat: Implement migration script for renaming timestamped files to content-hash based names with deduplication options
This commit is contained in:
Ross
@@ -249,23 +249,28 @@ async def run(args):
|
||||
try:
|
||||
parsed = json.loads(txt)
|
||||
pretty = json.dumps(parsed, ensure_ascii=False, indent=2)
|
||||
fname = f"{safe}_{h}_{ts}.json"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, pretty)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
content_bytes = pretty.encode('utf-8')
|
||||
excerpt = pretty[:800]
|
||||
use_text = pretty
|
||||
ext = 'json'
|
||||
except Exception:
|
||||
fname = f"{safe}_{h}_{ts}.txt"
|
||||
body_path = str(output / fname)
|
||||
save_text(output / fname, txt)
|
||||
content_bytes = txt.encode('utf-8')
|
||||
excerpt = txt[:800]
|
||||
use_text = txt
|
||||
ext = 'txt'
|
||||
|
||||
# deterministic filename based on content hash (sha1)
|
||||
content_sha1 = hashlib.sha1(content_bytes).hexdigest()
|
||||
fname = f"{safe}_{content_sha1}.{ext}"
|
||||
body_path = str(output / fname)
|
||||
|
||||
# if file already exists, don't rewrite body; just update meta/index
|
||||
if not os.path.exists(body_path):
|
||||
save_text(output / fname, use_text)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
excerpt = txt[:800]
|
||||
else:
|
||||
try:
|
||||
data = await resp.body()
|
||||
@@ -278,26 +283,29 @@ async def run(args):
|
||||
ext = guess_ext(ctype)
|
||||
# Put images into an images/ subdirectory, other binaries into assets/
|
||||
if ctype.startswith('image/'):
|
||||
# Images don't need per-capture timestamps; use stable names with hash
|
||||
subdir = output / 'images'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}.{ext}"
|
||||
content_sha1 = hashlib.sha1(data).hexdigest()
|
||||
fname = f"{safe}_{content_sha1}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
if not os.path.exists(body_path):
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
subdir = output / 'assets'
|
||||
subdir.mkdir(parents=True, exist_ok=True)
|
||||
fname = f"{safe}_{h}_{ts}.{ext}"
|
||||
content_sha1 = hashlib.sha1(data).hexdigest()
|
||||
fname = f"{safe}_{content_sha1}.{ext}"
|
||||
body_path = str(subdir / fname)
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
if not os.path.exists(body_path):
|
||||
save_binary(subdir / fname, data)
|
||||
try:
|
||||
print(f"{now_ts()}\tSAVED\t{body_path}")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
# final fallback: try binary
|
||||
try:
|
||||
@@ -340,11 +348,12 @@ async def run(args):
|
||||
except Exception:
|
||||
content_hash = None
|
||||
|
||||
# prepare the meta filename we'll write for this response
|
||||
new_meta_name = f"{safe}_{h}_{ts}.meta.json"
|
||||
# prepare the meta filename we'll write for this response (canonical by content when available)
|
||||
meta_base = content_hash or h
|
||||
new_meta_name = f"{safe}_{meta_base}.meta.json"
|
||||
new_meta_abs = os.path.abspath(str(output / new_meta_name))
|
||||
|
||||
# If older responses exist with the same content, delete them (fast in-memory lookup)
|
||||
# If older responses exist with the same content, handle them according to dedupe policy
|
||||
try:
|
||||
matches = in_memory_index.get(content_hash, []) if content_hash else []
|
||||
for j in matches:
|
||||
@@ -352,20 +361,55 @@ async def run(args):
|
||||
metaf = j.get('meta_file')
|
||||
try:
|
||||
if body and os.path.exists(body) and os.path.abspath(body) != os.path.abspath(body_path):
|
||||
os.remove(body)
|
||||
if args.dry_run:
|
||||
print(f"DRYRUN: would remove old body {body}")
|
||||
else:
|
||||
if args.dedupe_policy == 'delete':
|
||||
os.remove(body)
|
||||
elif args.dedupe_policy == 'move':
|
||||
dupdir = output / 'duplicates'
|
||||
dupdir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.replace(body, dupdir / os.path.basename(body))
|
||||
except Exception:
|
||||
os.remove(body)
|
||||
elif args.dedupe_policy == 'symlink':
|
||||
# replace old body with symlink to canonical
|
||||
try:
|
||||
os.remove(body)
|
||||
os.symlink(os.path.abspath(body_path), body)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tDELETED_OLD_BODY\t{body}\n")
|
||||
dbg.write(f"{now_ts()}\tDEDUP_BODY\t{body}\n")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if metaf and os.path.exists(metaf) and os.path.abspath(metaf) != new_meta_abs:
|
||||
os.remove(metaf)
|
||||
if args.dry_run:
|
||||
print(f"DRYRUN: would remove old meta {metaf}")
|
||||
else:
|
||||
if args.dedupe_policy == 'delete':
|
||||
os.remove(metaf)
|
||||
elif args.dedupe_policy == 'move':
|
||||
dupdir = output / 'duplicates'
|
||||
dupdir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.replace(metaf, dupdir / os.path.basename(metaf))
|
||||
except Exception:
|
||||
os.remove(metaf)
|
||||
elif args.dedupe_policy == 'symlink':
|
||||
try:
|
||||
os.remove(metaf)
|
||||
os.symlink(new_meta_abs, metaf)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
|
||||
dbg.write(f"{now_ts()}\tDELETED_OLD_META\t{metaf}\n")
|
||||
dbg.write(f"{now_ts()}\tDEDUP_META\t{metaf}\n")
|
||||
except Exception:
|
||||
pass
|
||||
except Exception:
|
||||
@@ -383,10 +427,10 @@ async def run(args):
|
||||
'response_excerpt': excerpt,
|
||||
'content_hash': content_hash,
|
||||
}
|
||||
meta_name = f"{safe}_{h}_{ts}.meta.json"
|
||||
save_text(output / meta_name, json.dumps(meta, ensure_ascii=False, indent=2))
|
||||
# write canonical meta (update if exists)
|
||||
save_text(output / new_meta_name, json.dumps(meta, ensure_ascii=False, indent=2))
|
||||
try:
|
||||
entry = {'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt, 'content_hash': content_hash}
|
||||
entry = {'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / new_meta_name), 'excerpt': excerpt, 'content_hash': content_hash}
|
||||
with open(index_path, 'a', encoding='utf-8') as idx:
|
||||
idx.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||||
# update in-memory index
|
||||
@@ -485,6 +529,8 @@ def parse_args():
|
||||
parser.add_argument('--post-login-selector', default='#ds-app', help='Selector that indicates a successful login')
|
||||
parser.add_argument('--channel', default=os.getenv('PLAYWRIGHT_CHROME_CHANNEL', 'chrome'))
|
||||
parser.add_argument('--capture-types', default='xhr,fetch,document,other')
|
||||
parser.add_argument('--dedupe-policy', default='delete', choices=['delete', 'move', 'keep', 'symlink'], help="What to do with older duplicate files: delete/move/keep/symlink")
|
||||
parser.add_argument('--dry-run', action='store_true', help='If set, show dedupe actions without performing them')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user