add who guideline scraper with pdf full text extraction

datasets/README.md

@@ -5,3 +5,21 @@ Dataset ingestion, processing, and construction for the [Agentic Medical Fact Ve
 Covers ingesting and normalizing source corpora, generating synthetic data with frontier / top open models, building train / validation / test splits, and producing fact-database entries.
 
 Workspace member (`amfv-datasets`).
+
+## Optional dependencies
+
+Sources that publish guidelines as PDFs need the `pdf` extra, which installs
+Docling for PDF-to-markdown conversion:
+
+```bash
+uv sync --group pdf
+```
+
+Without it those scrapers still run but fall back to whatever HTML the landing
+page exposes. See
+[scraping/LICENSE_NOTES.md](amfv_datasets/scraping/LICENSE_NOTES.md) for how
+`metadata.content_scope` reports this.
+
+Sample WHO guideline PDFs and the markdown extracted from them are committed in
+`test/fixtures/pdf/`. See [benchmarks/README.md](benchmarks/README.md) for how
+to inspect and verify the conversion, and why Docling is the default backend.

datasets/amfv_datasets/scraping/LICENSE_NOTES.md

@@ -0,0 +1,62 @@
+# Source licensing notes for scraped corpora
+
+## WHO (World Health Organization)
+
+Publications on [who.int](https://www.who.int/publications) published since
+November 2016 are licensed under **Creative Commons Attribution-NonCommercial-
+ShareAlike 3.0 IGO** (CC BY-NC-SA 3.0 IGO).
+
+- **Non-commercial use and adaptation** are permitted.
+- **Attribution** to WHO is required.
+- **Share-alike**: derivatives must use the same or a similar licence.
+
+Pre-2017 publications were not reissued under this licence. Use
+`metadata.publication_date` to filter when building corpora.
+
+### Suggested attribution
+
+> © World Health Organization {year}. *{publication title}*.
+> Licensed under CC BY-NC-SA 3.0 IGO.
+> https://creativecommons.org/licenses/by-nc-sa/3.0/igo/
+
+Each scraped document also records `metadata.license` and
+`metadata.attribution`.
+
+### Content scope
+
+The WHO scraper emits the HTML **Overview** from the publication landing page
+followed by the guideline body converted from the linked PDF
+(`metadata.download_url`). `metadata.content_scope` records which was captured:
+
+- `full` — Overview plus converted PDF body. `metadata.pdf_backend` and
+  `metadata.pdf_bytes` describe the conversion.
+- `overview` — Overview only, because no PDF was linked, the download failed, or
+  conversion failed. Both PDF metadata fields are `null`.
+
+PDF conversion needs the optional `pdf` extra (`uv sync --group pdf`). Without
+it every document degrades to `overview`, so check `content_scope` before
+treating a corpus as full text.
+
+### Committed test fixtures
+
+`datasets/test/fixtures/pdf/` holds six-page excerpts of two WHO guidelines, so
+conversion can be tested against real guideline text offline:
+
+| Fixture | Source | Pages |
+| --- | --- | --- |
+| `who_9789240121805_excerpt.pdf` | [Guidelines for the prevention of bloodstream infections and other infections associated with the use of intravascular catheters: part 2: central venous catheters](https://www.who.int/publications/i/item/9789240121805) | 30-35 |
+| `who_9789240124233_excerpt.pdf` | [Consolidated HIV guidelines: service delivery](https://www.who.int/publications/i/item/9789240124233) | 11-16 |
+
+Each PDF sits next to a `.expected.md` file holding the markdown the scraper
+extracts from it, so the conversion can be inspected without running anything.
+Those markdown files are derivative works of the PDFs and carry the same licence.
+
+Both are redistributed unmodified apart from page selection, under CC BY-NC-SA
+3.0 IGO:
+
+> © World Health Organization. Licensed under CC BY-NC-SA 3.0 IGO.
+> https://creativecommons.org/licenses/by-nc-sa/3.0/igo/
+
+They are test data for non-commercial research use. Note that this licence is
+more restrictive than the repository's Apache-2.0 licence, which covers the code
+only and does not extend to these files.

datasets/amfv_datasets/scraping/__init__.py

@@ -27,28 +27,56 @@
     scrape_guideline,
     scrape_nice,
 )
+from amfv_datasets.scraping.pdf import (
+    PdfBackend,
+    PdfConversionError,
+    count_markdown_sections,
+    pdf_to_markdown,
+)
+from amfv_datasets.scraping.who import (
+    WhoFetchError,
+    WhoListingPage,
+    WhoPublicationRef,
+    build_publication_text,
+    list_publications,
+    publication_ref_from_url,
+    scrape_publication,
+    scrape_who,
+)
 
 __all__ = [
     "GuidanceRef",
     "GuidanceListingPage",
     "LinkMode",
     "NiceFetchError",
     "OutputFormat",
+    "PdfBackend",
+    "PdfConversionError",
     "ScrapeError",
     "ScrapeRun",
     "ScrapedDocument",
     "ScraperSource",
     "USER_AGENT",
+    "WhoFetchError",
+    "WhoListingPage",
+    "WhoPublicationRef",
     "absolute_unique_urls",
     "build_guideline_text",
+    "build_publication_text",
     "clean_text",
+    "count_markdown_sections",
     "document_title",
     "default_client",
     "first_matching_urls",
     "guidance_ref_from_url",
     "html_to_markdown",
     "list_published_guidance",
+    "list_publications",
+    "pdf_to_markdown",
+    "publication_ref_from_url",
     "scrape_guideline",
     "scrape_listing_documents",
     "scrape_nice",
+    "scrape_publication",
+    "scrape_who",
 ]

datasets/amfv_datasets/scraping/cli.py

@@ -27,13 +27,15 @@
 from amfv_datasets.scraping.base import ScrapedDocument, ScrapeRun
 from amfv_datasets.scraping.html import LinkMode
 from amfv_datasets.scraping.nice import scrape_nice
+from amfv_datasets.scraping.who import scrape_who
 
 
 class ScraperSource(StrEnum):
     """Supported scraper sources."""
 
     ALL = "all"
     NICE = "nice"
+    WHO = "who"
 
 
 class OutputFormat(StrEnum):
@@ -72,6 +74,8 @@ def scrape_documents(
         match selected_source:
             case ScraperSource.NICE:
                 return scrape_nice(documents=documents, link_mode=link_mode, url=url)
+            case ScraperSource.WHO:
+                return scrape_who(documents=documents, link_mode=link_mode, url=url)
             case ScraperSource.ALL:
                 raise AssertionError("expanded source cannot be all")
     raise AssertionError(f"unsupported source: {source}")
@@ -125,7 +129,7 @@ def write_markdown_files(documents: Iterable[ScrapedDocument], output_path: Path
 
 def _expand_source(source: ScraperSource) -> tuple[ScraperSource, ...]:
     if source is ScraperSource.ALL:
-        return (ScraperSource.NICE,)
+        return (ScraperSource.NICE, ScraperSource.WHO)
     return (source,)
 
 

datasets/amfv_datasets/scraping/pdf.py

@@ -0,0 +1,169 @@
+"""Convert source PDFs into markdown for scrapers.
+
+Some guideline publishers release the document body only as a PDF, so the HTML
+page carries an abstract at best. These helpers convert such PDFs into the same
+clean markdown the HTML scrapers produce.
+
+Docling is the default backend. On a five-document sample of WHO guidelines it
+preserved GRADE evidence-table rows that Marker dropped, emitted no stray page
+numbers where PyMuPDF4LLM emitted roughly twenty per excerpt, and recovered the
+most heading structure. It also runs on CPU, which keeps the scrapers usable
+without a GPU. See `datasets/benchmarks/README.md` for the measurements and
+`datasets/benchmarks/pdf_backends.py` to reproduce them.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from enum import StrEnum
+from io import BytesIO
+
+from amfv_datasets.scraping.base import ScrapeError
+
+try:
+    from docling.datamodel.base_models import DocumentStream, InputFormat
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+    from docling.document_converter import DocumentConverter, PdfFormatOption
+
+    HAS_DOCLING = True
+except ImportError:
+    HAS_DOCLING = False
+
+logger = logging.getLogger(__name__)
+
+MIN_RUNNING_HEADER_CHARS = 20
+"""Shortest title fragment treated as a running header rather than content."""
+
+_HEADING_RE = re.compile(r"^(#{1,6})\s+\S")
+_BLANK_LINES_RE = re.compile(r"\n{3,}")
+
+
+class PdfBackend(StrEnum):
+    """Supported PDF-to-markdown conversion backends."""
+
+    DOCLING = "docling"
+
+
+class PdfConversionError(ScrapeError):
+    """Raised when a PDF cannot be converted to markdown."""
+
+
+_DOCLING_CONVERTERS: dict[bool, DocumentConverter] = {}
+
+
+def _docling_converter(*, ocr: bool) -> DocumentConverter:
+    """Return a cached Docling converter, since model load is expensive."""
+    converter = _DOCLING_CONVERTERS.get(ocr)
+    if converter is None:
+        options = PdfPipelineOptions()
+        options.do_ocr = ocr
+        options.do_table_structure = True
+        options.table_structure_options.do_cell_matching = True
+        converter = DocumentConverter(format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=options)})
+        _DOCLING_CONVERTERS[ocr] = converter
+    return converter
+
+
+def pdf_to_markdown(
+    data: bytes,
+    *,
+    backend: PdfBackend = PdfBackend.DOCLING,
+    ocr: bool = False,
+    running_header: str | None = None,
+    name: str = "document.pdf",
+) -> str:
+    """Convert PDF bytes into markdown.
+
+    Args:
+        data: Raw PDF bytes.
+        backend: Conversion backend to use (default: PdfBackend.DOCLING).
+        ocr: Whether to run OCR. Leave disabled for born-digital PDFs, where it
+            only adds runtime (default: False).
+        running_header: Repeated page header to drop from the output, usually the
+            document title (default: None).
+        name: File name reported to the backend, used for logging and format
+            detection (default: "document.pdf").
+    """
+    if not data:
+        raise PdfConversionError("Cannot convert an empty PDF payload")
+    if backend is not PdfBackend.DOCLING:
+        raise PdfConversionError(f"Unsupported PDF backend: {backend}")
+    if not HAS_DOCLING:
+        raise PdfConversionError("Docling is required to convert PDFs; install it with `uv sync --extra pdf`")
+
+    try:
+        result = _docling_converter(ocr=ocr).convert(DocumentStream(name=name, stream=BytesIO(data)))
+        markdown = result.document.export_to_markdown()
+    except Exception as exc:  # noqa: BLE001 - backend raises library-specific errors
+        raise PdfConversionError(f"Could not convert PDF '{name}': {exc}") from exc
+
+    markdown = _strip_running_header(markdown, running_header)
+    if not markdown.strip():
+        raise PdfConversionError(f"PDF '{name}' produced no readable markdown")
+    return markdown
+
+
+def count_markdown_sections(markdown: str) -> int:
+    """Count sections in markdown by its shallowest heading level.
+
+    Args:
+        markdown: Markdown text to inspect.
+    """
+    levels = [len(match.group(1)) for line in markdown.splitlines() if (match := _HEADING_RE.match(line.strip()))]
+    if not levels:
+        return 1
+    top_level = min(levels)
+    return sum(1 for level in levels if level == top_level)
+
+
+def _collapse_repeated_text(text: str) -> str:
+    """Reduce text built from a repeated phrase to a single copy.
+
+    Left and right page headers often land on one line, so a running header
+    arrives doubled.
+    """
+    words = text.split()
+    for parts in (2, 3, 4):
+        if len(words) < parts or len(words) % parts:
+            continue
+        size = len(words) // parts
+        chunk = words[:size]
+        if all(words[index * size : (index + 1) * size] == chunk for index in range(parts)):
+            return " ".join(chunk)
+    return text
+
+
+def _strip_running_header(markdown: str, running_header: str | None) -> str:
+    """Drop standalone lines repeating a PDF running header.
+
+    A line is dropped only when it is a long prefix of the title or the title is
+    a prefix of it, because publishers shorten the title in page headers. Lines
+    such as "(Strong recommendation, moderate-certainty evidence)" repeat
+    legitimately and must survive.
+    """
+    if not running_header:
+        return markdown.strip()
+
+    needle = " ".join(running_header.split()).casefold()
+    if not needle:
+        return markdown.strip()
+
+    kept: list[str] = []
+    for line in markdown.splitlines():
+        candidate = _collapse_repeated_text(" ".join(line.split()).casefold().strip("# "))
+        is_header = len(candidate) >= MIN_RUNNING_HEADER_CHARS and (
+            needle.startswith(candidate) or candidate.startswith(needle)
+        )
+        if not is_header:
+            kept.append(line)
+    return _BLANK_LINES_RE.sub("\n\n", "\n".join(kept)).strip()
+
+
+__all__ = [
+    "HAS_DOCLING",
+    "PdfBackend",
+    "PdfConversionError",
+    "count_markdown_sections",
+    "pdf_to_markdown",
+]