diff --git a/datasets/amfv_datasets/scraping/cli.py b/datasets/amfv_datasets/scraping/cli.py index acdcd050..0aac5143 100644 --- a/datasets/amfv_datasets/scraping/cli.py +++ b/datasets/amfv_datasets/scraping/cli.py @@ -27,6 +27,7 @@ from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.medlineplus import scrape_medlineplus from amfv_datasets.scraping.nice import scrape_nice @@ -41,6 +42,7 @@ def __call__(self, *, documents: int | None, link_mode: LinkMode, url: str | Non ALL_SOURCES = "all" SCRAPERS: dict[str, Scraper] = { + "medlineplus": scrape_medlineplus, "nice": scrape_nice, } """Scraper entry point by source name. Adding a source is an import and an entry here.""" diff --git a/datasets/amfv_datasets/scraping/medlineplus.py b/datasets/amfv_datasets/scraping/medlineplus.py new file mode 100644 index 00000000..a215e250 --- /dev/null +++ b/datasets/amfv_datasets/scraping/medlineplus.py @@ -0,0 +1,248 @@ +"""Scrape MedlinePlus health topic summaries into normalized markdown documents. + +MedlinePlus (U.S. National Library of Medicine) publishes clinician-reviewed +health topic summaries. Topics are discovered from the sitemap that +MedlinePlus advertises in its own robots.txt, then each topic page is scraped +for its summary and Dublin Core metadata. + +MedlinePlus also publishes a daily bulk XML export of every topic, which would +be one request instead of one per topic. We deliberately do not use it: it is +served from `/xml/`, which MedlinePlus robots.txt disallows. The topic pages +and the sitemap are both allowed, and the pages carry the same summary text +plus MeSH headings, alternate titles, and creation dates as metadata. + +Summary content lives in a `topic-summary` container that NLM marks +`syndicate`, its convention for content offered for reuse. Pages without that +container (indexes, tools, directories) are not health topics and are skipped. + +Licensing: NLM places health topic summaries in the public domain and allows +redistribution with the acknowledgement carried on every scraped document as +`ATTRIBUTION`. Content NLM licenses from third parties is deliberately out of +scope: A.D.A.M. encyclopedia articles (`/ency/`) and ASHP drug monographs +(`/druginfo/`) cannot be redistributed without licensing from those vendors, +and neither matches the flat topic path this scraper accepts. See +https://medlineplus.gov/about/using/usingcontent/. + +Documents are short: live samples run 500-6,500 characters, median ~1,600, +since these are patient-facing summaries rather than clinical guidelines. For +comparison, the NICE scraper's guidelines run a median of 23,817 characters. +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Iterable +from urllib.parse import urlparse + +import httpx +from lxml import etree +from lxml import html as lxml_html + +from amfv_datasets.scraping.base import ( + ScrapedDocument, + ScrapeError, + ScrapeRun, + default_client, + scrape_listing_documents, +) +from amfv_datasets.scraping.html import LinkMode, clean_text, html_to_markdown + +BASE_URL = "https://medlineplus.gov" +SITEMAP_URL = f"{BASE_URL}/sitemap.xml" +MEDLINEPLUS_DATASET_NAME = "medlineplus-webscrape" +MEDLINEPLUS_DATASET_DISPLAY_NAME = "MedlinePlus Webscrape" +ATTRIBUTION = "Courtesy of MedlinePlus from the National Library of Medicine" +"""Acknowledgement NLM asks redistributors of public domain content to carry.""" +DOCUMENT_DELAY_SECONDS = 1.0 +"""Delay between topic pages. MedlinePlus robots.txt sets no Crawl-delay, so +this is a politeness floor rather than a required interval.""" + +logger = logging.getLogger(__name__) + +_SITEMAP_NS = {"sitemap": "http://www.sitemaps.org/schemas/sitemap/0.9"} +# English topic pages are a single flat slug, e.g. /a1c.html. Anything nested +# (/spanish/..., /ency/..., /druginfo/...) is a translation or another corpus. +_TOPIC_PATH_RE = re.compile(r"^/(?P[a-z0-9]+)\.html$") + + +class MedlineplusFetchError(ScrapeError): + """Raised when a MedlinePlus topic cannot be fetched or parsed.""" + + +def topic_slug_from_url(url: str) -> str: + """Return the topic slug for a MedlinePlus topic URL. + + Args: + url: MedlinePlus topic URL, e.g. `https://medlineplus.gov/a1c.html`. + """ + parsed = urlparse(url.strip()) + if parsed.scheme not in {"http", "https"} or parsed.netloc.lower() not in { + "medlineplus.gov", + "www.medlineplus.gov", + }: + raise MedlineplusFetchError(f"Enter a MedlinePlus topic URL from medlineplus.gov; got {url!r}") + match = _TOPIC_PATH_RE.match(parsed.path) + if not match: + raise MedlineplusFetchError(f"Enter an English topic URL like {BASE_URL}/a1c.html; got {url!r}") + return match.group("slug") + + +def list_topic_urls(client: httpx.Client) -> list[str]: + """Return candidate English topic page URLs from the MedlinePlus sitemap. + + The sitemap also lists indexes, tools, and directory pages that share the + topic URL shape; those are filtered out when scraped, since only real + topics carry a summary container. + + Args: + client: HTTP client used to fetch the sitemap. + """ + try: + response = client.get(SITEMAP_URL) + response.raise_for_status() + except httpx.HTTPError as exc: + raise MedlineplusFetchError(f"Could not fetch the MedlinePlus sitemap at {SITEMAP_URL}") from exc + try: + root = etree.fromstring(response.content) + except etree.XMLSyntaxError as exc: + raise MedlineplusFetchError(f"Could not parse the MedlinePlus sitemap at {SITEMAP_URL}") from exc + + urls: list[str] = [] + seen: set[str] = set() + for location in root.findall(".//sitemap:url/sitemap:loc", _SITEMAP_NS): + url = (location.text or "").strip() + if not url or url in seen: + continue + parsed = urlparse(url) + if parsed.netloc.lower() != "medlineplus.gov" or not _TOPIC_PATH_RE.match(parsed.path): + continue + seen.add(url) + urls.append(url) + if not urls: + raise MedlineplusFetchError(f"No topic URLs found in the MedlinePlus sitemap at {SITEMAP_URL}") + return urls + + +def scrape_topic(client: httpx.Client, url: str, *, link_mode: LinkMode = LinkMode.KEEP) -> ScrapedDocument | None: + """Scrape one MedlinePlus topic page into a normalized document. + + Returns None when the page carries no summary container, which is how + non-topic pages in the sitemap are skipped, or when the page could not be + fetched, which is logged rather than raised so one unreachable page in a + large crawl costs a document instead of the whole run. + + Args: + client: HTTP client used to fetch the topic page. + url: MedlinePlus topic URL to scrape. + link_mode: Whether links are kept as markdown links or stripped to + their visible text (default: LinkMode.KEEP). + """ + try: + response = client.get(url) + response.raise_for_status() + except httpx.HTTPError as exc: + logger.warning("Skipping MedlinePlus topic %s: %s", url, exc) + return None + doc = lxml_html.fromstring(response.text) + + summaries = doc.xpath('//div[@id="topic-summary"]') + if not summaries: + return None + content = html_to_markdown(lxml_html.tostring(summaries[0], encoding="unicode"), link_mode=link_mode, base_url=url) + if not content: + return None + + title = _meta_value(doc, "DC.Title") or _first_text(doc, '//div[@id="topic"]//h1//text()') + if not title: + return None + + return ScrapedDocument( + source="medlineplus", + external_id=f"medlineplus-{topic_slug_from_url(url)}", + title=title, + url=url, + content=content, + metadata={ + "attribution": ATTRIBUTION, + "also_called": _meta_values(doc, "DC.Title.Alternate"), + "date_created": _meta_value(doc, "DC.Date.Created"), + "date_modified": _meta_value(doc, "DC.Date.Modified"), + "mesh_headings": _meta_values(doc, "DC.Subject.MeSH"), + "publisher": _meta_value(doc, "DC.Publisher"), + }, + ) + + +def scrape_medlineplus( + *, + documents: int | None, + link_mode: LinkMode = LinkMode.KEEP, + url: str | None = None, +) -> ScrapeRun: + """Scrape MedlinePlus health topics discovered from the sitemap. + + Args: + documents: Number of topics to scrape. Ignored when `url` is set. When + unset, every topic in the sitemap is scraped (default: None). + link_mode: Whether links are kept as markdown links or stripped to + their visible text (default: LinkMode.KEEP). + url: MedlinePlus topic URL to scrape as a single document (default: + None). + """ + if url is not None: + topic_url = f"{BASE_URL}/{topic_slug_from_url(url)}.html" + + def scrape_url() -> Iterable[ScrapedDocument]: + with default_client() as client: + document = scrape_topic(client, topic_url, link_mode=link_mode) + if document is None: + raise MedlineplusFetchError(f"No topic summary found at '{topic_url}'") + yield document + + return ScrapeRun(documents=scrape_url(), total=1) + + with default_client() as client: + topic_urls = list_topic_urls(client) + + return ScrapeRun( + total=documents if documents is not None else len(topic_urls), + documents=scrape_listing_documents( + documents=documents, + client_factory=default_client, + first_page_items=topic_urls, + list_page=lambda client, page: [], + scrape_item=lambda client, topic_url: scrape_topic(client, topic_url, link_mode=link_mode), + document_delay_seconds=DOCUMENT_DELAY_SECONDS, + ), + ) + + +def _meta_values(doc: lxml_html.HtmlElement, name: str) -> list[str]: + values = [clean_text(value) for value in doc.xpath(f'//meta[@name="{name}"]/@content')] + return [value for value in values if value] + + +def _meta_value(doc: lxml_html.HtmlElement, name: str) -> str: + values = _meta_values(doc, name) + return values[0] if values else "" + + +def _first_text(doc: lxml_html.HtmlElement, xpath: str) -> str: + values = [clean_text(value) for value in doc.xpath(xpath)] + return next((value for value in values if value), "") + + +__all__ = [ + "ATTRIBUTION", + "BASE_URL", + "DOCUMENT_DELAY_SECONDS", + "MEDLINEPLUS_DATASET_DISPLAY_NAME", + "MEDLINEPLUS_DATASET_NAME", + "SITEMAP_URL", + "MedlineplusFetchError", + "list_topic_urls", + "scrape_medlineplus", + "scrape_topic", + "topic_slug_from_url", +] diff --git a/datasets/test/test_scraping_cli.py b/datasets/test/test_scraping_cli.py index bb24a7f6..4f3bd207 100644 --- a/datasets/test/test_scraping_cli.py +++ b/datasets/test/test_scraping_cli.py @@ -229,7 +229,7 @@ def test_cli_run_rejects_an_unregistered_source() -> None: assert result.exit_code != 0 assert "'nhs'" in result.stderr - assert "all, nice" in result.stderr + assert "all, medlineplus, nice" in result.stderr def test_expand_source_runs_every_registered_scraper() -> None: diff --git a/datasets/test/test_scraping_medlineplus.py b/datasets/test/test_scraping_medlineplus.py new file mode 100644 index 00000000..9c79e671 --- /dev/null +++ b/datasets/test/test_scraping_medlineplus.py @@ -0,0 +1,290 @@ +"""Tests for MedlinePlus scraping helpers.""" + +import logging + +import httpx +import pytest +from typer.testing import CliRunner + +from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun +from amfv_datasets.scraping.cli import SCRAPERS, app +from amfv_datasets.scraping.html import LinkMode +from amfv_datasets.scraping.medlineplus import ( + BASE_URL, + MedlineplusFetchError, + list_topic_urls, + scrape_medlineplus, + scrape_topic, + topic_slug_from_url, +) + +_TOPIC_HTML = """ + + + + + + + + + + + +
+

A1C

+
+

A1C tests for type 2 diabetes.

+
+
+ + +""" + +_TOPIC_HTML_ASTHMA = """ + + + +
+

Asthma

+

Asthma affects the airways.

+
+ + +""" + +_NON_TOPIC_HTML = """ + +

Health Check Tools

An index page.

+ +""" + +_SITEMAP_XML = """ + + https://medlineplus.gov/a1c.html + https://medlineplus.gov/healthchecktools.html + https://medlineplus.gov/a1c.html + https://medlineplus.gov/spanish/a1c.html + https://medlineplus.gov/ency/article/003640.htm + https://medlineplus.gov/druginfo/herb_All.html + https://medlineplus.gov/lab-tests/a1c-test/ + +""" + + +def test_list_topic_urls_keeps_only_flat_english_topic_pages() -> None: + """Translations, nested corpora, and duplicate sitemap entries are filtered out.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/sitemap.xml" + return httpx.Response(200, content=_SITEMAP_XML.encode("utf-8")) + + client = httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL) + + assert list_topic_urls(client) == [ + "https://medlineplus.gov/a1c.html", + "https://medlineplus.gov/healthchecktools.html", + ] + + +def test_list_topic_urls_raises_when_the_sitemap_has_no_topics() -> None: + """An unexpected sitemap shape fails loudly rather than scraping nothing.""" + empty = b'' + + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, content=empty))) + + with pytest.raises(MedlineplusFetchError, match="No topic URLs"): + list_topic_urls(client) + + +def test_scrape_topic_converts_summary_and_collects_metadata() -> None: + """A topic page becomes a normalized document with markdown content.""" + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/a1c.html" + return httpx.Response(200, text=_TOPIC_HTML) + + client = httpx.Client(transport=httpx.MockTransport(handler)) + + document = scrape_topic(client, "https://medlineplus.gov/a1c.html") + + assert document is not None + assert document.source == "medlineplus" + assert document.external_id == "medlineplus-a1c" + assert document.title == "A1C" + assert document.url == "https://medlineplus.gov/a1c.html" + assert document.content == "A1C tests for [type 2 diabetes](https://medlineplus.gov/diabetestype2.html)." + assert document.metadata == { + "attribution": "Courtesy of MedlinePlus from the National Library of Medicine", + "also_called": ["Hemoglobin A1c", "HbA1c"], + "date_created": "2015-12-22", + "date_modified": "2026-08-01", + "mesh_headings": ["Glycated Hemoglobin"], + "publisher": "National Library of Medicine", + } + + +def test_scrape_topic_strips_links_in_strip_mode() -> None: + """Link stripping mode drops markdown link syntax from the summary.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, text=_TOPIC_HTML))) + + document = scrape_topic(client, "https://medlineplus.gov/a1c.html", link_mode=LinkMode.STRIP) + + assert document is not None + assert document.content == "A1C tests for type 2 diabetes." + + +def test_scrape_topic_skips_pages_without_a_summary() -> None: + """Index and tool pages that share the topic URL shape are skipped.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(200, text=_NON_TOPIC_HTML))) + + assert scrape_topic(client, "https://medlineplus.gov/healthchecktools.html") is None + + +def test_scrape_topic_skips_and_logs_a_page_that_fails_to_fetch(caplog: pytest.LogCaptureFixture) -> None: + """One unreachable page is skipped rather than aborting the whole crawl.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(503))) + + with caplog.at_level(logging.WARNING): + document = scrape_topic(client, "https://medlineplus.gov/a1c.html") + + assert document is None + assert "a1c.html" in caplog.text + + +def test_list_topic_urls_raises_medlineplus_fetch_error_when_the_sitemap_fails() -> None: + """A sitemap fetch failure raises this module's error type, not a raw httpx error.""" + client = httpx.Client(transport=httpx.MockTransport(lambda request: httpx.Response(503))) + + with pytest.raises(MedlineplusFetchError, match="Could not fetch"): + list_topic_urls(client) + + +@pytest.mark.parametrize( + ("url", "expected_message"), + [ + ("https://example.com/a1c.html", "from medlineplus.gov"), + ("https://medlineplus.gov/spanish/a1c.html", "English topic URL"), + ("https://medlineplus.gov/ency/article/003640.htm", "English topic URL"), + ], + ids=["wrong-host", "spanish-translation", "encyclopedia-article"], +) +def test_topic_slug_from_url_rejects_non_topic_urls(url: str, expected_message: str) -> None: + """Non-topic URLs are rejected with a message naming the expected shape.""" + with pytest.raises(MedlineplusFetchError, match=expected_message): + topic_slug_from_url(url) + + +def test_topic_slug_from_url_accepts_a_topic_url() -> None: + """A topic URL resolves to its slug.""" + assert topic_slug_from_url("https://medlineplus.gov/a1c.html") == "a1c" + + +@pytest.mark.parametrize( + "url", + [ + "https://medlineplus.gov/ency/article/003640.htm", + "https://medlineplus.gov/druginfo/meds/a693048.html", + ], + ids=["adam-encyclopedia", "ashp-drug-monograph"], +) +def test_third_party_licensed_corpora_are_never_scraped(url: str) -> None: + """NLM licenses /ency/ and /druginfo/ from vendors, so they stay out of the corpus.""" + sitemap = ( + '' + f"{url}" + ) + client = httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=sitemap.encode("utf-8"))) + ) + + # Excluded from discovery, and rejected outright when passed as --url. + with pytest.raises(MedlineplusFetchError, match="No topic URLs"): + list_topic_urls(client) + with pytest.raises(MedlineplusFetchError, match="English topic URL"): + topic_slug_from_url(url) + + +def test_scrape_medlineplus_scrapes_every_topic_and_fetches_the_sitemap_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The SCRAPERS entry point discovers topics from the sitemap and scrapes each one.""" + sitemap = ( + '' + "https://medlineplus.gov/a1c.html" + "https://medlineplus.gov/asthma.html" + "" + ) + request_paths: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + request_paths.append(request.url.path) + if request.url.path == "/sitemap.xml": + return httpx.Response(200, content=sitemap.encode("utf-8")) + if request.url.path == "/a1c.html": + return httpx.Response(200, text=_TOPIC_HTML) + if request.url.path == "/asthma.html": + return httpx.Response(200, text=_TOPIC_HTML_ASTHMA) + raise AssertionError(f"unexpected request to {request.url}") + + # scrape_medlineplus opens a fresh client per default_client() call (one for + # discovery, one inside scrape_listing_documents), so this must be a factory + # rather than a single shared client, which scrape_listing_documents closes + # after its own use. + monkeypatch.setattr( + "amfv_datasets.scraping.medlineplus.default_client", + lambda: httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL), + ) + monkeypatch.setattr("amfv_datasets.scraping.base.time.sleep", lambda seconds: None) + + scrape_run = scrape_medlineplus(documents=None, link_mode=LinkMode.KEEP) + documents = list(scrape_run.documents) + + assert scrape_run.total == 2 + assert {document.title for document in documents} == {"A1C", "Asthma"} + assert request_paths.count("/sitemap.xml") == 1 + + +def test_scrape_medlineplus_url_mode_normalizes_a_www_host(monkeypatch: pytest.MonkeyPatch) -> None: + """A www.-prefixed --url still resolves to the canonical medlineplus.gov page.""" + request_paths: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + request_paths.append(request.url.path) + assert request.url.host == "medlineplus.gov" + return httpx.Response(200, text=_TOPIC_HTML) + + monkeypatch.setattr( + "amfv_datasets.scraping.medlineplus.default_client", + lambda: httpx.Client(transport=httpx.MockTransport(handler), base_url=BASE_URL), + ) + + scrape_run = scrape_medlineplus(documents=None, url="https://www.medlineplus.gov/a1c.html") + documents = list(scrape_run.documents) + + assert scrape_run.total == 1 + assert len(documents) == 1 + assert documents[0].url == "https://medlineplus.gov/a1c.html" + assert request_paths == ["/a1c.html"] + + +def test_cli_dispatches_the_medlineplus_source(monkeypatch: pytest.MonkeyPatch) -> None: + """--source medlineplus resolves through the CLI's SCRAPERS registry to this module.""" + + def fake_scrape_medlineplus(*, documents: int | None, link_mode: LinkMode, url: str | None = None) -> ScrapeRun: + assert documents == 2 + assert url is None + document = ScrapedDocument( + source="medlineplus", + external_id="medlineplus-a1c", + title="A1C", + url="https://medlineplus.gov/a1c.html", + content="content", + ) + return ScrapeRun([document], total=1) + + monkeypatch.setitem(SCRAPERS, "medlineplus", fake_scrape_medlineplus) + + result = CliRunner().invoke(app, ["--source", "medlineplus", "--documents", "2", "--no-progress"]) + + assert result.exit_code == 0 + assert '"external_id": "medlineplus-a1c"' in result.stdout