Enhance document processing in scrapers

- Updated pediatric multiple sclerosis article reference ID.
- Improved document_to_markdown.py to handle differential diagnoses, tables, anatomy, and cases more effectively.
- Added logging for cached entries of DDX, Tables, Anatomy, and Cases.
- Adjusted file pattern matching to process all JSON files by default.
- Fixed document ID extraction logic for various data types.

docs_md/_images_index.json

@@ -63,7 +63,6 @@
   "docs_md/articles/adem_a3fafeb7-5861-4364-beb8-c0e30220564e.md": [
     "images/app.statdx.com_image_deba3310-8e09-466c-97d2-1e6bccb28edf_dafbadc1_20251018T064947Z.jpg",
     "images/app.statdx.com_image_thumbnail_deba3310-8e09-466c-97d2-1e6bccb28edf_size_168_quality_85_3dfef960_20251018T064936Z.jpg",
-    "images/app.statdx.com_image_thumbnail_deba3310-8e09-466c-97d2-1e6bccb28edf_size_174_quality_85_6f9ba8b5_20251018T122441Z.jpg",
     "images/app.statdx.com_image_e18458d3-5f6e-43fd-ab6d-72c7a86dca92_0f3893d6_20251018T064949Z.jpg",
     "images/app.statdx.com_image_thumbnail_e18458d3-5f6e-43fd-ab6d-72c7a86dca92_size_168_quality_85_70272c40_20251018T064936Z.jpg",
     "images/app.statdx.com_image_53135c0f-4c9e-4fb9-ab09-df46c5b40f70_c986514a_20251018T064950Z.jpg",
@@ -352,7 +351,7 @@
   ],
   "docs_md/articles/basal-ganglia-calcification_f8dc8f27-f256-480d-9393-7ec3495a3d27.md": [
     "images/app.statdx.com_image_thumbnail_04a1f1d8-6210-4030-84ff-4f60f3f7f7bf_annotated_true_size_900_quality_90_ab12a15f_20251018T121530Z.jpg",
-    "images/app.statdx.com_image_thumbnail_04a1f1d8-6210-4030-84ff-4f60f3f7f7bf_size_174_quality_85_63332bdc_20251018T115114Z.jpg",
+    "images/app.statdx.com_image_thumbnail_04a1f1d8-6210-4030-84ff-4f60f3f7f7bf_size_174_quality_85_02fa8f20_20251018T142330Z.jpg",
     "images/app.statdx.com_image_thumbnail_da569666-3c25-4be0-9f3d-339aa20a7c0c_annotated_true_size_900_quality_90_69540ccd_20251018T121530Z.jpg",
     "images/app.statdx.com_image_thumbnail_b0b9b464-052e-445c-8496-65170c3de33e_annotated_true_size_900_quality_90_826e4674_20251018T121530Z.jpg",
     "images/app.statdx.com_image_thumbnail_0cde205f-c504-4cff-a434-767e3b3a2ae8_annotated_true_size_900_quality_90_a68440f2_20251018T121530Z.jpg",
@@ -420,5 +419,71 @@
     "images/app.statdx.com_image_thumbnail_5a2a2ea2-f45b-4465-b4d2-ba5685cc617d_annotated_true_size_900_quality_90_40187927_20251018T124348Z.jpg",
     "images/app.statdx.com_image_thumbnail_b6035566-14cd-44af-95a0-de102374633d_annotated_true_size_900_quality_90_562d9ced_20251018T124348Z.jpg",
     "images/app.statdx.com_image_thumbnail_71506ec8-ddb5-4665-88cd-1a9bbbb6a4fd_annotated_true_size_900_quality_90_8e155fb2_20251018T124348Z.jpg"
+  ],
+  "docs_md/articles/abnormal-shape-configuration-of-corpus-callosum_238ca32d-6bc6-4f5a-81b1-6601dd605856.md": [
+    "images/app.statdx.com_image_thumbnail_770c0ebd-7c6f-4197-a62b-704c6b7d6bb6_annotated_true_size_900_quality_90_56dafce6_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_770c0ebd-7c6f-4197-a62b-704c6b7d6bb6_size_174_quality_85_d61e2530_20251018T125015Z.jpg",
+    "images/app.statdx.com_image_thumbnail_259efe0c-d735-48aa-a8a4-102adb752c04_annotated_true_size_900_quality_90_44e855c1_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_fdffb43b-acf7-4645-ae14-03f4bc8c461c_annotated_true_size_900_quality_90_a9dd954a_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_419e3589-55cb-4187-abe4-34701b078398_annotated_true_size_900_quality_90_521cfe60_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_601e0a1e-2d56-4c42-8215-e2d7a9357c1c_annotated_true_size_900_quality_90_0ed7b3c3_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_62a0bdfd-2c15-4578-8548-30dbd6278948_annotated_true_size_900_quality_90_665f8b2a_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_a124c2db-747c-4ab8-885e-95a07831790d_annotated_true_size_900_quality_90_967e57cd_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_2f23b513-918a-4b07-b08f-a03f25edaae2_annotated_true_size_900_quality_90_cd311d46_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_b0668126-045a-4375-a13d-8336f6f18cc8_annotated_true_size_900_quality_90_ef6279e6_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_62167fbf-3d3c-4c29-b456-b457c903d157_annotated_true_size_900_quality_90_27e4bdc5_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_5b3bafb7-d57f-4111-bd8b-1a122c74d4e2_annotated_true_size_900_quality_90_9b2efa1d_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_dd083e9e-beac-4e49-8c01-df6dcb4e4e72_annotated_true_size_900_quality_90_e58b8787_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_82c23417-cc0d-47b9-a72c-9253390c8669_annotated_true_size_900_quality_90_7081e987_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_5ba333fc-5752-4158-9879-eb9fba5607f7_annotated_true_size_900_quality_90_67a96337_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_f86d2115-e3b9-421e-ad82-19c308b0bbe4_annotated_true_size_900_quality_90_ba2a85fb_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_acee92fd-3247-482d-97d6-df3fe96c1863_annotated_true_size_900_quality_90_246ff2ac_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_d7178b25-99a0-48e5-b5d1-4ba745e31ffd_annotated_true_size_900_quality_90_c0b535a4_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_4e887fcd-b71a-4ea0-9a32-a0c92831bd3c_annotated_true_size_900_quality_90_3b5f0c6f_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_254a4187-4098-4461-a132-5f320ed115cc_annotated_true_size_900_quality_90_4d84f230_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_2cdae2ee-21a2-4d37-86b8-ec8c63300332_annotated_true_size_900_quality_90_def17346_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_87401e50-17f7-4c3a-a13d-25585cf8587c_annotated_true_size_900_quality_90_3fda2aba_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_26c5225d-e501-4ac2-8b64-dcad9220c0cc_annotated_true_size_900_quality_90_9af45eb7_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_f06b5e0c-4fa2-4141-90e9-b3ff276df084_annotated_true_size_900_quality_90_02851778_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_45dda608-de0b-4b0c-b752-7a53664c3cf2_annotated_true_size_900_quality_90_5f982aeb_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_ab751550-37f4-4c06-8aa3-704d77ade93f_annotated_true_size_900_quality_90_edf1566f_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_5b506ba1-f44e-4b24-bd43-d4b66ddd0f74_annotated_true_size_900_quality_90_51994411_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_4febdaa6-7169-4512-a441-b7ab601c6290_annotated_true_size_900_quality_90_98e8fade_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_2148930a-f910-449f-8f04-9b59c8919068_annotated_true_size_900_quality_90_4e19bec0_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_46f727cc-b7d9-4ffc-a993-b5616685d0c1_annotated_true_size_900_quality_90_3e8787d9_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_702f029e-c743-4233-a437-58f7445a29ce_annotated_true_size_900_quality_90_7e055645_20251018T142218Z.jpg",
+    "images/app.statdx.com_image_thumbnail_8d204964-2869-4537-b851-89072fefa22b_annotated_true_size_900_quality_90_08fdf3d7_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_3122fe3d-7f2a-4446-b348-8f14efa4f4a2_annotated_true_size_900_quality_90_5c9ae416_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_efe29b1b-0a22-4a79-a164-306b4f9d09cc_annotated_true_size_900_quality_90_a66d8937_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_554904c6-222f-4f20-b05e-3581543671af_annotated_true_size_900_quality_90_878a7aa0_20251018T142240Z.jpg",
+    "images/app.statdx.com_image_thumbnail_45d959b2-20ac-493b-a132-b5a1c2354b55_annotated_true_size_900_quality_90_0e22670a_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_d5ade043-0a79-4e06-a806-2a6fd3d1509f_annotated_true_size_900_quality_90_3a494990_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_126a8ad4-dc8d-4abf-832d-87fd23880c01_annotated_true_size_900_quality_90_5dbfb155_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_82054e81-0249-4fb3-8216-0739f38441a7_annotated_true_size_900_quality_90_7162eb50_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_4cc06316-602d-4975-b9dd-ad30fd02d052_annotated_true_size_900_quality_90_cc24fcf3_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_28f4c3c1-8c1c-4e6c-8815-4824e939b53e_annotated_true_size_900_quality_90_4a62f400_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_0ecf80f2-8db2-4f59-a30b-4b1b5b71ab10_annotated_true_size_900_quality_90_32c3bdbd_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_4e121da9-31c4-476f-8048-b690b404c2e8_annotated_true_size_900_quality_90_2fe43416_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_106f9fc3-f917-413c-b862-0fca057f2e8b_annotated_true_size_900_quality_90_345ea79c_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_cd7db346-bbb1-4ace-8084-e6b609ffd3a4_annotated_true_size_900_quality_90_1324558b_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_807b1f61-4777-4bc3-8f59-ecdd2b8d484d_annotated_true_size_900_quality_90_42f971bd_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_92ac059a-cc5f-46fb-bcec-54b20c03f7d2_annotated_true_size_900_quality_90_9f582aba_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_f6e3e0fd-f4ab-4e5c-a4d0-2f1abdb32e13_annotated_true_size_900_quality_90_29400ff9_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_99d1900d-a440-4c87-a4d7-a63b771cb20e_annotated_true_size_900_quality_90_093fee8d_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_05a43764-0e56-43a7-a397-c6985ccf2f90_annotated_true_size_900_quality_90_fb43a056_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_65c833a7-4e18-443c-9f90-1a5b1878c31e_annotated_true_size_900_quality_90_e3eebedd_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_c3835c36-b514-487d-9e87-fb2c809411b8_annotated_true_size_900_quality_90_3cb56218_20251018T142242Z.jpg",
+    "images/app.statdx.com_image_thumbnail_e972264a-0edb-41ef-827c-f0a12c2068f5_annotated_true_size_900_quality_90_a49b1938_20251018T142242Z.jpg"
+  ],
+  "docs_md/articles/multiple-sclerosis_89599954-599e-4410-a517-eb22125cedfb.md": [
+    "images/app.statdx.com_image_thumbnail_cdd1d309-5f88-4517-8434-da8f015ecf81_annotated_true_size_900_quality_90_d1dfab0c_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_f7c6cfd4-7a27-4173-b60d-f813b61cf506_annotated_true_size_900_quality_90_94245c62_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_6bfb1493-e878-43b1-a089-407b3f0efbba_annotated_true_size_900_quality_90_28a87781_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_b72c256f-7f30-4ff0-9b03-2848dbebf832_annotated_true_size_900_quality_90_8f2ca444_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_5d6a0b48-ad9b-482b-99e5-bb3acf61fd6d_annotated_true_size_900_quality_90_d2dc5fbb_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_65ddf85c-133c-416b-987b-f64f8360baea_annotated_true_size_900_quality_90_9dd80371_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_927e7fcc-08ba-4599-a057-8c54ea03bfad_annotated_true_size_900_quality_90_c2a635e9_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_ddda2a05-d65f-416c-9100-45a824c86f76_annotated_true_size_900_quality_90_a8ef1ebd_20251018T142350Z.jpg",
+    "images/app.statdx.com_image_thumbnail_e26b711f-9edd-4e0b-bf62-14e7fdc5e6e5_annotated_true_size_900_quality_90_c6c3c128_20251018T142350Z.jpg"
   ]
 }