From 6b067a21215b7cc59f9c5d013c6308a98068998b Mon Sep 17 00:00:00 2001 From: Valmi Date: Tue, 25 Aug 2026 22:15:09 -0400 Subject: [PATCH 1/2] add medlineplus scraper Discovers topics from the sitemap MedlinePlus advertises in its robots.txt, then scrapes each topic page for its summary and Dublin Core metadata (MeSH headings, alternate titles, dates). Deliberately avoids the daily bulk XML export, which would be a single request but is served from /xml/, a path MedlinePlus robots.txt disallows. Pages that share the topic URL shape but carry no summary container are skipped, so indexes and tools do not enter the corpus. Health topic summaries are public domain, and each document carries the acknowledgement NLM asks redistributors for. Content NLM licenses from third parties is excluded by construction: A.D.A.M. articles and ASHP drug monographs live off the flat topic path this scraper accepts. MedlinePlus is listed as an encyclopedic-fallback source in the AMFV project doc. --- datasets/amfv_datasets/scraping/cli.py | 2 + .../amfv_datasets/scraping/medlineplus.py | 232 ++++++++++++++++++ datasets/test/test_scraping_cli.py | 2 +- datasets/test/test_scraping_medlineplus.py | 167 +++++++++++++ 4 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 datasets/amfv_datasets/scraping/medlineplus.py create mode 100644 datasets/test/test_scraping_medlineplus.py diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index acdcd050..0aac5143 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -27,6 +27,7 @@ from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.medlineplus import scrape_medlineplus from amfv_datasets.scraping.nice import scrape_nice @@ -41,6 +42,7 @@ def __call__(self, *, documents: int | None, link_mode: LinkMode, url: str | Non ALL_SOURCES = "all" SCRAPERS: dict[str, Scraper] = { + "medlineplus": scrape_medlineplus, "nice": scrape_nice, } """Scraper entry point by source name. Adding a source is an import and an entry here.""" diff --git a/datasets/amfv_datasets/scraping/medlineplus.py b/datasets/amfv_datasets/scraping/medlineplus.py new file mode 100644 index 00000000..bdff331b --- /dev/null +++ b/datasets/amfv_datasets/scraping/medlineplus.py @@ -0,0 +1,232 @@ +"""Scrape MedlinePlus health topic summaries into normalized markdown documents. + +MedlinePlus (U.S. National Library of Medicine) publishes clinician-reviewed +health topic summaries. Topics are discovered from the sitemap that +MedlinePlus advertises in its own robots.txt, then each topic page is scraped +for its summary and Dublin Core metadata. + +MedlinePlus also publishes a daily bulk XML export of every topic, which would +be one request instead of one per topic. We deliberately do not use it: it is +served from `/xml/`, which MedlinePlus robots.txt disallows. The topic pages +and the sitemap are both allowed, and the pages carry the same summary text +plus MeSH headings, alternate titles, and creation dates as metadata. + +Summary content lives in a `topic-summary` container that NLM marks +`syndicate`, its convention for content offered for reuse. Pages without that +container (indexes, tools, directories) are not health topics and are skipped. + +Licensing: NLM places health topic summaries in the public domain and allows +redistribution with the acknowledgement carried on every scraped document as +`ATTRIBUTION`. Content NLM licenses from third parties is deliberately out of +scope: A.D.A.M. encyclopedia articles (`/ency/`) and ASHP drug monographs +(`/druginfo/`) cannot be redistributed without licensing from those vendors, +and neither matches the flat topic path this scraper accepts. See +https://medlineplus.gov/about/using/usingcontent/. +""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from urllib.parse import urlparse + +import httpx +from lxml import etree +from lxml import html as lxml_html + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, clean_text, html_to_markdown + +BASE_URL = "https://medlineplus.gov" +SITEMAP_URL = f"{BASE_URL}/sitemap.xml" +MEDLINEPLUS_DATASET_NAME = "medlineplus-webscrape" +MEDLINEPLUS_DATASET_DISPLAY_NAME = "MedlinePlus Webscrape" +ATTRIBUTION = "Courtesy of MedlinePlus from the National Library of Medicine" +"""Acknowledgement NLM asks redistributors of public domain content to carry.""" +DOCUMENT_DELAY_SECONDS = 1.0 +"""Delay between topic pages. MedlinePlus robots.txt sets no Crawl-delay, so +this is a politeness floor rather than a required interval.""" + +_SITEMAP_NS = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"} +# English topic pages are a single flat slug, e.g. /a1c.html. Anything nested +# (/spanish/..., /ency/..., /druginfo/...) is a translation or another corpus. +_TOPIC_PATH_RE = re.compile(r"^/(?P[a-z0-9]+)\.html$") + + +class MedlineplusFetchError(ScrapeError): + """Raised when a MedlinePlus topic cannot be fetched or parsed.""" + + +def topic_slug_from_url(url: str) -> str: + """Return the topic slug for a MedlinePlus topic URL. + + Args: + url: MedlinePlus topic URL, e.g. `https://medlineplus.gov/a1c.html`. + """ + parsed = urlparse(url.strip()) + if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in { + "medlineplus.gov", + "www.medlineplus.gov", + }: + raise MedlineplusFetchError(f"Enter a MedlinePlus topic URL from medlineplus.gov; got {url!r}") + match = _TOPIC_PATH_RE.match(parsed.path) + if not match: + raise MedlineplusFetchError(f"Enter an English topic URL like {BASE_URL}/a1c.html; got {url!r}") + return match.group("slug") + + +def list_topic_urls(client: httpx.Client) -> list[str]: + """Return candidate English topic page URLs from the MedlinePlus sitemap. + + The sitemap also lists indexes, tools, and directory pages that share the + topic URL shape; those are filtered out when scraped, since only real + topics carry a summary container. + + Args: + client: HTTP client used to fetch the sitemap. + """ + response = client.get(SITEMAP_URL) + response.raise_for_status() + try: + root = etree.fromstring(response.content) + except etree.XMLSyntaxError as exc: + raise MedlineplusFetchError(f"Could not parse the MedlinePlus sitemap at {SITEMAP_URL}") from exc + + urls: list[str] = [] + seen: set[str] = set() + for location in root.findall(".//sitemap:url/sitemap:loc", _SITEMAP_NS): + url = (location.text or "").strip() + if not url or url in seen: + continue + parsed = urlparse(url) + if parsed.netloc.lower() != "medlineplus.gov" or not _TOPIC_PATH_RE.match(parsed.path): + continue + seen.add(url) + urls.append(url) + if not urls: + raise MedlineplusFetchError(f"No topic URLs found in the MedlinePlus sitemap at {SITEMAP_URL}") + return urls + + +def scrape_topic(client: httpx.Client, url: str, *, link_mode: LinkMode = LinkMode.KEEP) -> ScrapedDocument | None: + """Scrape one MedlinePlus topic page into a normalized document. + + Returns None when the page carries no summary container, which is how + non-topic pages in the sitemap are skipped. + + Args: + client: HTTP client used to fetch the topic page. + url: MedlinePlus topic URL to scrape. + link_mode: Whether links are kept as markdown links or stripped to + their visible text (default: LinkMode.KEEP). + """ + response = client.get(url) + response.raise_for_status() + doc = lxml_html.fromstring(response.text) + + summaries = doc.xpath('//div[@id="topic-summary"]') + if not summaries: + return None + content = html_to_markdown(lxml_html.tostring(summaries[0], encoding="unicode"), link_mode=link_mode, base_url=url) + if not content: + return None + + title = _meta_value(doc, "DC.Title") or _first_text(doc, '//div[@id="topic"]//h1//text()') + if not title: + return None + + return ScrapedDocument( + source="medlineplus", + external_id=f"medlineplus-{topic_slug_from_url(url)}", + title=title, + url=url, + content=content, + metadata={ + "attribution": ATTRIBUTION, + "also_called": _meta_values(doc, "DC.Title.Alternate"), + "date_created": _meta_value(doc, "DC.Date.Created"), + "date_modified": _meta_value(doc, "DC.Date.Modified"), + "mesh_headings": _meta_values(doc, "DC.Subject.MeSH"), + "publisher": _meta_value(doc, "DC.Publisher"), + }, + ) + + +def scrape_medlineplus( + *, + documents: int | None, + link_mode: LinkMode = LinkMode.KEEP, + url: str | None = None, +) -> ScrapeRun: + """Scrape MedlinePlus health topics discovered from the sitemap. + + Args: + documents: Number of topics to scrape. Ignored when `url` is set. When + unset, every topic in the sitemap is scraped (default: None). + link_mode: Whether links are kept as markdown links or stripped to + their visible text (default: LinkMode.KEEP). + url: MedlinePlus topic URL to scrape as a single document (default: + None). + """ + if url is not None: + topic_url = f"{BASE_URL}/{topic_slug_from_url(url)}.html" + + def scrape_url() -> Iterable[ScrapedDocument]: + with default_client() as client: + document = scrape_topic(client, topic_url, link_mode=link_mode) + if document is None: + raise MedlineplusFetchError(f"No topic summary found at '{topic_url}'") + yield document + + return ScrapeRun(documents=scrape_url(), total=1) + + with default_client() as client: + topic_urls = list_topic_urls(client) + + return ScrapeRun( + total=documents, + documents=scrape_listing_documents( + documents=documents, + client_factory=default_client, + first_page_items=topic_urls, + list_page=lambda client, page: [], + scrape_item=lambda client, topic_url: scrape_topic(client, topic_url, link_mode=link_mode), + document_delay_seconds=DOCUMENT_DELAY_SECONDS, + ), + ) + + +def _meta_values(doc: lxml_html.HtmlElement, name: str) -> list[str]: + values = [clean_text(value) for value in doc.xpath(f'//meta[@name="{name}"]/@content')] + return [value for value in values if value] + + +def _meta_value(doc: lxml_html.HtmlElement, name: str) -> str: + values = _meta_values(doc, name) + return values[0] if values else "" + + +def _first_text(doc: lxml_html.HtmlElement, xpath: str) -> str: + values = [clean_text(value) for value in doc.xpath(xpath)] + return next((value for value in values if value), "") + + +__all__ = [ + "ATTRIBUTION", + "BASE_URL", + "DOCUMENT_DELAY_SECONDS", + "MEDLINEPLUS_DATASET_DISPLAY_NAME", + "MEDLINEPLUS_DATASET_NAME", + "SITEMAP_URL", + "MedlineplusFetchError", + "list_topic_urls", + "scrape_medlineplus", + "scrape_topic", + "topic_slug_from_url", +] diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py index bb24a7f6..4f3bd207 100644 --- a/datasets/test/test_scraping_cli.py +++ b/datasets/test/test_scraping_cli.py @@ -229,7 +229,7 @@ def test_cli_run_rejects_an_unregistered_source() -> None: assert result.exit_code != 0 assert "'nhs'" in result.stderr - assert "all, nice" in result.stderr + assert "all, medlineplus, nice" in result.stderr def test_expand_source_runs_every_registered_scraper() -> None: diff --git a/datasets/test/test_scraping_medlineplus.py b/datasets/test/test_scraping_medlineplus.py new file mode 100644 index 00000000..34a58b6c --- /dev/null +++ b/datasets/test/test_scraping_medlineplus.py @@ -0,0 +1,167 @@ +"""Tests for MedlinePlus scraping helpers.""" + +import httpx +import pytest + +from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.medlineplus import ( + BASE_URL, + MedlineplusFetchError, + list_topic_urls, + scrape_topic, + topic_slug_from_url, +) + +_TOPIC_HTML = """ + + + + + + + + + + + +
+

A1C

+
+

A1C tests for type 2 diabetes.

+
+
+ + +""" + +_NON_TOPIC_HTML = """ + +

Health Check Tools

An index page.

+ +""" + +_SITEMAP_XML = """ + + https://medlineplus.gov/a1c.html + https://medlineplus.gov/healthchecktools.html + https://medlineplus.gov/a1c.html + https://medlineplus.gov/spanish/a1c.html + https://medlineplus.gov/ency/article/003640.htm + https://medlineplus.gov/druginfo/herb_All.html + https://medlineplus.gov/lab-tests/a1c-test/ + +""" + + +def test_list_topic_urls_keeps_only_flat_english_topic_pages() -> None: + """Translations, nested corpora, and duplicate sitemap entries are filtered out.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/sitemap.xml" + return httpx.Response(200, content=_SITEMAP_XML.encode("utf-8")) + + client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) + + assert list_topic_urls(client) == [ + "https://medlineplus.gov/a1c.html", + "https://medlineplus.gov/healthchecktools.html", + ] + + +def test_list_topic_urls_raises_when_the_sitemap_has_no_topics() -> None: + """An unexpected sitemap shape fails loudly rather than scraping nothing.""" + empty = b'' + + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, content=empty))) + + with pytest.raises(MedlineplusFetchError, match="No topic URLs"): + list_topic_urls(client) + + +def test_scrape_topic_converts_summary_and_collects_metadata() -> None: + """A topic page becomes a normalized document with markdown content.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/a1c.html" + return httpx.Response(200, text=_TOPIC_HTML) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + + document = scrape_topic(client, "https://medlineplus.gov/a1c.html") + + assert document is not None + assert document.source == "medlineplus" + assert document.external_id == "medlineplus-a1c" + assert document.title == "A1C" + assert document.url == "https://medlineplus.gov/a1c.html" + assert document.content == "A1C tests for [type 2 diabetes](https://medlineplus.gov/diabetestype2.html)." + assert document.metadata == { + "attribution": "Courtesy of MedlinePlus from the National Library of Medicine", + "also_called": ["Hemoglobin A1c", "HbA1c"], + "date_created": "2015-12-22", + "date_modified": "2026-08-01", + "mesh_headings": ["Glycated Hemoglobin"], + "publisher": "National Library of Medicine", + } + + +def test_scrape_topic_strips_links_in_strip_mode() -> None: + """Link stripping mode drops markdown link syntax from the summary.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, text=_TOPIC_HTML))) + + document = scrape_topic(client, "https://medlineplus.gov/a1c.html", link_mode=LinkMode.STRIP) + + assert document is not None + assert document.content == "A1C tests for type 2 diabetes." + + +def test_scrape_topic_skips_pages_without_a_summary() -> None: + """Index and tool pages that share the topic URL shape are skipped.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, text=_NON_TOPIC_HTML))) + + assert scrape_topic(client, "https://medlineplus.gov/healthchecktools.html") is None + + +@pytest.mark.parametrize( + ("url", "expected_message"), + [ + ("https://example.com/a1c.html", "from medlineplus.gov"), + ("https://medlineplus.gov/spanish/a1c.html", "English topic URL"), + ("https://medlineplus.gov/ency/article/003640.htm", "English topic URL"), + ], + ids=["wrong-host", "spanish-translation", "encyclopedia-article"], +) +def test_topic_slug_from_url_rejects_non_topic_urls(url: str, expected_message: str) -> None: + """Non-topic URLs are rejected with a message naming the expected shape.""" + with pytest.raises(MedlineplusFetchError, match=expected_message): + topic_slug_from_url(url) + + +def test_topic_slug_from_url_accepts_a_topic_url() -> None: + """A topic URL resolves to its slug.""" + assert topic_slug_from_url("https://medlineplus.gov/a1c.html") == "a1c" + + +@pytest.mark.parametrize( + "url", + [ + "https://medlineplus.gov/ency/article/003640.htm", + "https://medlineplus.gov/druginfo/meds/a693048.html", + ], + ids=["adam-encyclopedia", "ashp-drug-monograph"], +) +def test_third_party_licensed_corpora_are_never_scraped(url: str) -> None: + """NLM licenses /ency/ and /druginfo/ from vendors, so they stay out of the corpus.""" + sitemap = ( + '' + f"{url}" + ) + client = httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=sitemap.encode("utf-8"))) + ) + + # Excluded from discovery, and rejected outright when passed as --url. + with pytest.raises(MedlineplusFetchError, match="No topic URLs"): + list_topic_urls(client) + with pytest.raises(MedlineplusFetchError, match="English topic URL"): + topic_slug_from_url(url) From 7a120db0f5f08fac1f668a2506e508158d02cddd Mon Sep 17 00:00:00 2001 From: Valmi Date: Wed, 2 Sep 2026 17:01:15 -0400 Subject: [PATCH 2/2] address review: don't abort the whole crawl on one bad page Per @zndr27's review on #23: - list_topic_urls and scrape_topic now raise/return MedlinePlusFetchError- wrapped failures instead of a raw httpx error escaping. The sitemap fetch stays fatal (discovery can't proceed without it); a single topic page fetch failure is logged and skipped, since scrape_listing_documents already treats None as "skip this item". A 503 partway through a run now costs one document instead of the rest of the crawl. - total is now an upper bound from the discovered topic count instead of always None, so the progress bar has a denominator on a full run. - Documented typical document size next to the licensing paragraph. - Added the missing coverage: scrape_medlineplus end to end (including that the sitemap is fetched exactly once), the --url branch's www. host normalization, and a CLI dispatch test for the medlineplus source. --- .../amfv_datasets/scraping/medlineplus.py | 28 +++- datasets/test/test_scraping_medlineplus.py | 123 ++++++++++++++++++ 2 files changed, 145 insertions(+), 6 deletions(-) diff --git a/datasets/amfv_datasets/scraping/medlineplus.py b/datasets/amfv_datasets/scraping/medlineplus.py index bdff331b..a215e250 100644 --- a/datasets/amfv_datasets/scraping/medlineplus.py +++ b/datasets/amfv_datasets/scraping/medlineplus.py @@ -22,10 +22,15 @@ (`/druginfo/`) cannot be redistributed without licensing from those vendors, and neither matches the flat topic path this scraper accepts. See https://medlineplus.gov/about/using/usingcontent/. + +Documents are short: live samples run 500-6,500 characters, median ~1,600, +since these are patient-facing summaries rather than clinical guidelines. For +comparison, the NICE scraper's guidelines run a median of 23,817 characters. """ from __future__ import annotations +import logging import re from collections.abc import Iterable from urllib.parse import urlparse @@ -53,6 +58,8 @@ """Delay between topic pages. MedlinePlus robots.txt sets no Crawl-delay, so this is a politeness floor rather than a required interval.""" +logger = logging.getLogger(__name__) + _SITEMAP_NS = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"} # English topic pages are a single flat slug, e.g. /a1c.html. Anything nested # (/spanish/..., /ency/..., /druginfo/...) is a translation or another corpus. @@ -91,8 +98,11 @@ def list_topic_urls(client: httpx.Client) -> list[str]: Args: client: HTTP client used to fetch the sitemap. """ - response = client.get(SITEMAP_URL) - response.raise_for_status() + try: + response = client.get(SITEMAP_URL) + response.raise_for_status() + except httpx.HTTPError as exc: + raise MedlineplusFetchError(f"Could not fetch the MedlinePlus sitemap at {SITEMAP_URL}") from exc try: root = etree.fromstring(response.content) except etree.XMLSyntaxError as exc: @@ -118,7 +128,9 @@ def scrape_topic(client: httpx.Client, url: str, *, link_mode: LinkMode = LinkMo """Scrape one MedlinePlus topic page into a normalized document. Returns None when the page carries no summary container, which is how - non-topic pages in the sitemap are skipped. + non-topic pages in the sitemap are skipped, or when the page could not be + fetched, which is logged rather than raised so one unreachable page in a + large crawl costs a document instead of the whole run. Args: client: HTTP client used to fetch the topic page. @@ -126,8 +138,12 @@ def scrape_topic(client: httpx.Client, url: str, *, link_mode: LinkMode = LinkMo link_mode: Whether links are kept as markdown links or stripped to their visible text (default: LinkMode.KEEP). """ - response = client.get(url) - response.raise_for_status() + try: + response = client.get(url) + response.raise_for_status() + except httpx.HTTPError as exc: + logger.warning("Skipping MedlinePlus topic %s: %s", url, exc) + return None doc = lxml_html.fromstring(response.text) summaries = doc.xpath('//div[@id="topic-summary"]') @@ -190,7 +206,7 @@ def scrape_url() -> Iterable[ScrapedDocument]: topic_urls = list_topic_urls(client) return ScrapeRun( - total=documents, + total=documents if documents is not None else len(topic_urls), documents=scrape_listing_documents( documents=documents, client_factory=default_client, diff --git a/datasets/test/test_scraping_medlineplus.py b/datasets/test/test_scraping_medlineplus.py index 34a58b6c..9c79e671 100644 --- a/datasets/test/test_scraping_medlineplus.py +++ b/datasets/test/test_scraping_medlineplus.py @@ -1,13 +1,19 @@ """Tests for MedlinePlus scraping helpers.""" +import logging + import httpx import pytest +from typer.testing import CliRunner +from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun +from amfv_datasets.scraping.cli import SCRAPERS, app from amfv_datasets.scraping.html import LinkMode from amfv_datasets.scraping.medlineplus import ( BASE_URL, MedlineplusFetchError, list_topic_urls, + scrape_medlineplus, scrape_topic, topic_slug_from_url, ) @@ -34,6 +40,18 @@ """ +_TOPIC_HTML_ASTHMA = """ + + + +
+

Asthma

+

Asthma affects the airways.

+
+ + +""" + _NON_TOPIC_HTML = """

Health Check Tools

An index page.

@@ -122,6 +140,25 @@ def test_scrape_topic_skips_pages_without_a_summary() -> None: assert scrape_topic(client, "https://medlineplus.gov/healthchecktools.html") is None +def test_scrape_topic_skips_and_logs_a_page_that_fails_to_fetch(caplog: pytest.LogCaptureFixture) -> None: + """One unreachable page is skipped rather than aborting the whole crawl.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(503))) + + with caplog.at_level(logging.WARNING): + document = scrape_topic(client, "https://medlineplus.gov/a1c.html") + + assert document is None + assert "a1c.html" in caplog.text + + +def test_list_topic_urls_raises_medlineplus_fetch_error_when_the_sitemap_fails() -> None: + """A sitemap fetch failure raises this module's error type, not a raw httpx error.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(503))) + + with pytest.raises(MedlineplusFetchError, match="Could not fetch"): + list_topic_urls(client) + + @pytest.mark.parametrize( ("url", "expected_message"), [ @@ -165,3 +202,89 @@ def test_third_party_licensed_corpora_are_never_scraped(url: str) -> None: list_topic_urls(client) with pytest.raises(MedlineplusFetchError, match="English topic URL"): topic_slug_from_url(url) + + +def test_scrape_medlineplus_scrapes_every_topic_and_fetches_the_sitemap_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The SCRAPERS entry point discovers topics from the sitemap and scrapes each one.""" + sitemap = ( + '' + "https://medlineplus.gov/a1c.html" + "https://medlineplus.gov/asthma.html" + "" + ) + request_paths: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + request_paths.append(request.url.path) + if request.url.path == "/sitemap.xml": + return httpx.Response(200, content=sitemap.encode("utf-8")) + if request.url.path == "/a1c.html": + return httpx.Response(200, text=_TOPIC_HTML) + if request.url.path == "/asthma.html": + return httpx.Response(200, text=_TOPIC_HTML_ASTHMA) + raise AssertionError(f"unexpected request to {request.url}") + + # scrape_medlineplus opens a fresh client per default_client() call (one for + # discovery, one inside scrape_listing_documents), so this must be a factory + # rather than a single shared client, which scrape_listing_documents closes + # after its own use. + monkeypatch.setattr( + "amfv_datasets.scraping.medlineplus.default_client", + lambda: httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL), + ) + monkeypatch.setattr("amfv_datasets.scraping.base.time.sleep", lambda seconds: None) + + scrape_run = scrape_medlineplus(documents=None, link_mode=LinkMode.KEEP) + documents = list(scrape_run.documents) + + assert scrape_run.total == 2 + assert {document.title for document in documents} == {"A1C", "Asthma"} + assert request_paths.count("/sitemap.xml") == 1 + + +def test_scrape_medlineplus_url_mode_normalizes_a_www_host(monkeypatch: pytest.MonkeyPatch) -> None: + """A www.-prefixed --url still resolves to the canonical medlineplus.gov page.""" + request_paths: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + request_paths.append(request.url.path) + assert request.url.host == "medlineplus.gov" + return httpx.Response(200, text=_TOPIC_HTML) + + monkeypatch.setattr( + "amfv_datasets.scraping.medlineplus.default_client", + lambda: httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL), + ) + + scrape_run = scrape_medlineplus(documents=None, url="https://www.medlineplus.gov/a1c.html") + documents = list(scrape_run.documents) + + assert scrape_run.total == 1 + assert len(documents) == 1 + assert documents[0].url == "https://medlineplus.gov/a1c.html" + assert request_paths == ["/a1c.html"] + + +def test_cli_dispatches_the_medlineplus_source(monkeypatch: pytest.MonkeyPatch) -> None: + """--source medlineplus resolves through the CLI's SCRAPERS registry to this module.""" + + def fake_scrape_medlineplus(*, documents: int | None, link_mode: LinkMode, url: str | None = None) -> ScrapeRun: + assert documents == 2 + assert url is None + document = ScrapedDocument( + source="medlineplus", + external_id="medlineplus-a1c", + title="A1C", + url="https://medlineplus.gov/a1c.html", + content="content", + ) + return ScrapeRun([document], total=1) + + monkeypatch.setitem(SCRAPERS, "medlineplus", fake_scrape_medlineplus) + + result = CliRunner().invoke(app, ["--source", "medlineplus", "--documents", "2", "--no-progress"]) + + assert result.exit_code == 0 + assert '"external_id": "medlineplus-a1c"' in result.stdout