ross/statdx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add comprehensive documentation for adrenal conditions

Browse Source 
- Created detailed articles for Adrenal Adenoma, Adrenal Cyst, Adrenal Myelolipoma, and general Adrenal anatomy.
- Included key facts, imaging findings, differential diagnoses, pathology, clinical issues, and diagnostic checklists for each condition.
- Enhanced understanding of adrenal tumors and their characteristics through structured documentation.
This commit is contained in:
Ross
2025-10-17 22:27:59 +01:00
parent 2f8df5f820
commit 55716a16c5
38 changed files with 1232 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scrapers/capture_passive_playwright_async.py
+168 -1
View File
@@ -87,6 +87,30 @@ async def run(args):
await asyncio.sleep(5)
async with capture_count_lock:
print(f"[heartbeat] captured={capture_count}")
# Build an in-memory index of content_hash -> list of index entries for quick dedupe lookups.
# If older index lines lack a content_hash but reference a body file, hash that file once at startup.
in_memory_index = {}
try:
if index_path.exists():
with open(index_path, 'r', encoding='utf-8') as idxf:
for line in idxf:
try:
j = json.loads(line)
except Exception:
continue
ch = j.get('content_hash')
body = j.get('body_file')
if not ch and body and os.path.exists(body):
try:
with open(body, 'rb') as bf:
ch = hashlib.sha256(bf.read()).hexdigest()
except Exception:
ch = None
if ch:
in_memory_index.setdefault(ch, []).append(j)
except Exception:
in_memory_index = {}
async with async_playwright() as p:
browser_type = p.chromium
@@ -102,6 +126,82 @@ async def run(args):
page = await context.new_page()
async def attempt_autologin(page, username: str, password: str, post_login_selector: str, wait_after: float = 1.0):
"""Async form-based autologin similar to save_page_snapshots.attempt_autologin."""
if not username or not password:
return False
# selectors
user_selectors = ['input[name="username"]', 'input.usernameSelector', 'input[type="email"]', 'input[name*="email" i]', 'input[name*="user" i]', 'input[type="text"]']
pass_selectors = ['input[type="password"]', 'input[name="password"]', 'input.passwordSelector']
user_sel = None
for sel in user_selectors:
try:
el = await page.query_selector(sel)
if el:
user_sel = sel
break
except Exception:
continue
pass_sel = None
for sel in pass_selectors:
try:
el = await page.query_selector(sel)
if el:
pass_sel = sel
break
except Exception:
continue
if not user_sel or not pass_sel:
try:
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
dbg.write(f"{now_ts()}\tAUTOLOGIN\tselectors_missing\n")
except Exception:
pass
return False
try:
await page.fill(user_sel, username)
await asyncio.sleep(0.1)
await page.fill(pass_sel, password)
await asyncio.sleep(0.1)
# try submit
submit_selectors = ['button[type=submit]', 'input[type=submit]', 'button:has-text("Sign in")', 'button:has-text("Sign In")', 'button:has-text("Log in")']
clicked = False
for s in submit_selectors:
try:
btn = await page.query_selector(s)
if btn:
await btn.click()
clicked = True
break
except Exception:
continue
if not clicked:
try:
await page.press(pass_sel, 'Enter')
except Exception:
pass
# wait for post-login selector
try:
if post_login_selector:
await page.wait_for_selector(post_login_selector, timeout=10000)
except Exception:
pass
await asyncio.sleep(wait_after)
return True
except Exception:
try:
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
dbg.write(f"{now_ts()}\tAUTOLOGIN\tfailed_exception\n")
except Exception:
pass
return False
async def on_response(resp):
nonlocal capture_count
try:
@@ -237,6 +337,51 @@ async def run(args):
# skip if still failing
return
# compute content hash for dedupe
content_hash = None
try:
if 'txt' in locals() and isinstance(txt, str):
content_hash = hashlib.sha256(txt.encode('utf-8')).hexdigest()
elif 'pretty' in locals() and isinstance(pretty, str):
content_hash = hashlib.sha256(pretty.encode('utf-8')).hexdigest()
elif 'data' in locals() and isinstance(data, (bytes, bytearray)):
content_hash = hashlib.sha256(data).hexdigest()
except Exception:
content_hash = None
# prepare the meta filename we'll write for this response
new_meta_name = f"{safe}_{h}_{ts}.meta.json"
new_meta_abs = os.path.abspath(str(output / new_meta_name))
# If older responses exist with the same content, delete them (fast in-memory lookup)
try:
matches = in_memory_index.get(content_hash, []) if content_hash else []
for j in matches:
body = j.get('body_file')
metaf = j.get('meta_file')
try:
if body and os.path.exists(body) and os.path.abspath(body) != os.path.abspath(body_path):
os.remove(body)
try:
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
dbg.write(f"{now_ts()}\tDELETED_OLD_BODY\t{body}\n")
except Exception:
pass
except Exception:
pass
try:
if metaf and os.path.exists(metaf) and os.path.abspath(metaf) != new_meta_abs:
os.remove(metaf)
try:
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
dbg.write(f"{now_ts()}\tDELETED_OLD_META\t{metaf}\n")
except Exception:
pass
except Exception:
pass
except Exception:
pass
meta = {
'url': url,
'resource_type': rtype,
@@ -245,12 +390,17 @@ async def run(args):
'response_headers': resp_headers,
'response_body_file': body_path,
'response_excerpt': excerpt,
'content_hash': content_hash,
}
meta_name = f"{safe}_{h}_{ts}.meta.json"
save_text(output / meta_name, json.dumps(meta, ensure_ascii=False, indent=2))
try:
entry = {'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt, 'content_hash': content_hash}
with open(index_path, 'a', encoding='utf-8') as idx:
idx.write(json.dumps({'url': url, 'resource_type': rtype, 'timestamp': ts, 'body_file': body_path, 'meta_file': str(output / meta_name), 'excerpt': excerpt}, ensure_ascii=False) + '\n')
idx.write(json.dumps(entry, ensure_ascii=False) + '\n')
# update in-memory index
if content_hash:
in_memory_index.setdefault(content_hash, []).append(entry)
except Exception:
pass
@@ -290,6 +440,20 @@ async def run(args):
except Exception:
pass
# attempt autologin (form-based) if credentials provided
uname = args.username or os.getenv('STATDX_USERNAME')
pwd = args.password or os.getenv('STATDX_PASSWORD')
if uname and pwd:
try:
ok = await attempt_autologin(page, uname, pwd, args.post_login_selector)
try:
with open(output / '_debug_events.log', 'a', encoding='utf-8') as dbg:
dbg.write(f"{now_ts()}\tAUTOLOGIN_RESULT\t{ok}\n")
except Exception:
pass
except Exception:
pass
# If interactive, let user log in; otherwise start capture immediately
if not args.no_prompt:
print('When you have logged in in the opened browser, press Enter here to continue and capture...')
@@ -325,6 +489,9 @@ def parse_args():
parser.add_argument('--headless', action='store_true')
parser.add_argument('--continuous', action='store_true')
parser.add_argument('--no-prompt', action='store_true')
parser.add_argument('--username', help='STATdx username (or use STATDX_USERNAME env var)')
parser.add_argument('--password', help='STATdx password (or use STATDX_PASSWORD env var)')
parser.add_argument('--post-login-selector', default='#ds-app', help='Selector that indicates a successful login')
parser.add_argument('--channel', default=os.getenv('PLAYWRIGHT_CHROME_CHANNEL', 'chrome'))
parser.add_argument('--capture-types', default='xhr,fetch,document,other')
return parser.parse_args()