Add CCO scraper

.gitignore

@@ -216,3 +216,4 @@ __marimo__/
 
 # Streamlit
 .streamlit/secrets.toml
+data/

datasets/amfv_datasets/__init__.py

@@ -1 +1,74 @@
-"""Dataset processing utilities for AMFV."""
+"""Web scraping helpers and source-specific scrapers."""
+
+from amfv_datasets.scraping.base import (
+    USER_AGENT,
+    ScrapedDocument,
+    ScrapeError,
+    ScrapeRun,
+    default_client,
+    scrape_listing_documents,
+)
+from amfv_datasets.scraping.cco import (
+    CcoFetchError,
+    GuidelineRef,
+    PageFetch,
+    guideline_ref_from_url,
+    list_cco_guidelines,
+    playwright_client,
+    playwright_fetch,
+    scrape_cco,
+    scrape_cco_guideline,
+)
+from amfv_datasets.scraping.cli import OutputFormat, ScraperSource
+from amfv_datasets.scraping.html import (
+    LinkMode,
+    absolute_unique_urls,
+    clean_text,
+    document_title,
+    first_matching_urls,
+    html_to_markdown,
+)
+from amfv_datasets.scraping.nice import (
+    GuidanceListingPage,
+    GuidanceRef,
+    NiceFetchError,
+    build_guideline_text,
+    guidance_ref_from_url,
+    list_published_guidance,
+    scrape_guideline,
+    scrape_nice,
+)
+
+__all__ = [
+    "CcoFetchError",
+    "GuidanceRef",
+    "GuidanceListingPage",
+    "GuidelineRef",
+    "LinkMode",
+    "NiceFetchError",
+    "OutputFormat",
+    "PageFetch",
+    "ScrapeError",
+    "ScrapeRun",
+    "ScrapedDocument",
+    "ScraperSource",
+    "USER_AGENT",
+    "absolute_unique_urls",
+    "build_guideline_text",
+    "clean_text",
+    "document_title",
+    "default_client",
+    "first_matching_urls",
+    "guidance_ref_from_url",
+    "guideline_ref_from_url",
+    "html_to_markdown",
+    "list_cco_guidelines",
+    "list_published_guidance",
+    "playwright_client",
+    "playwright_fetch",
+    "scrape_cco",
+    "scrape_cco_guideline",
+    "scrape_guideline",
+    "scrape_listing_documents",
+    "scrape_nice",
+]

datasets/amfv_datasets/scraping/base.py

@@ -69,12 +69,12 @@
     return httpx.Client(headers=client_headers, timeout=timeout, follow_redirects=follow_redirects)
 
 
-def scrape_listing_documents[ListingItemT](
+def scrape_listing_documents[ClientT, ListingItemT](
     *,
     documents: int | None,
-    client_factory: Callable[[], AbstractContextManager[httpx.Client]],
-    list_page: Callable[[httpx.Client, int], Iterable[ListingItemT]],
-    scrape_item: Callable[[httpx.Client, ListingItemT], ScrapedDocument],
+    client_factory: Callable[[], AbstractContextManager[ClientT]],
+    list_page: Callable[[ClientT, int], Iterable[ListingItemT]],
+    scrape_item: Callable[[ClientT, ListingItemT], ScrapedDocument],
     document_delay_seconds: float = 5.0,
     first_page_items: Iterable[ListingItemT] | None = None,
 ) -> Iterable[ScrapedDocument]:
@@ -83,7 +83,8 @@
     Args:
         documents: Number of documents to scrape. When unset, listing pages are
             fetched until a page returns no items (default: None).
-        client_factory: Factory returning a context-managed HTTP client.
+        client_factory: Factory returning a context-managed client, passed through
+            to `list_page` and `scrape_item` unchanged.
         list_page: Function that lists source-specific items for a page.
         scrape_item: Function that scrapes one listed item into a document.
         document_delay_seconds: Delay before scraping each document after the
@@ -125,4 +126,4 @@
     "USER_AGENT",
     "default_client",
     "scrape_listing_documents",
-]
+]