From fb0da8704d73a7cb8dca63f8721f669ba0a49ef8 Mon Sep 17 00:00:00 2001 From: Yahia Bakour Date: Thu, 8 Oct 2026 12:48:15 -0400 Subject: [PATCH 1/3] docs: lead README with Context.dev capabilities --- README.md | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ddcab92..207f49a 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,7 @@ [![PyPI version](https://img.shields.io/pypi/v/context.dev.svg?label=pypi%20(stable))](https://pypi.org/project/context.dev/) -The Context Dev Python library provides convenient access to the Context Dev REST API from any Python 3.9+ -application. The library includes type definitions for all request params and response fields, -and offers both synchronous and asynchronous clients powered by [httpx](https://github.com/encode/httpx). +Context.dev is a web scraping API for AI agents and LLMs. This SDK turns any URL into clean, LLM-ready markdown, crawls whole sites, searches the web, takes screenshots and extracts structured JSON against a schema you define, all with one API key. Proxies, JavaScript rendering and anti-bot handling run on Context.dev's side, so there is no headless browser to host. It is generated with [Stainless](https://www.stainless.com/). @@ -39,6 +37,49 @@ brand = client.brand.retrieve( print(brand.request_id) ``` +### Extract structured JSON + +```python +from context.dev import ContextDev + +client = ContextDev() + +page = client.web.scrape( + url="https://example.com", + formats={"json": True}, + json_params={ + "schema": { + "type": "object", + "properties": { + "title": {"type": ["string", "null"]}, + "description": {"type": ["string", "null"]}, + }, + "required": ["title", "description"], + "additionalProperties": False, + }, + }, +) +print(page.json_.data) +``` + +## What you can do + +| Task | Method | +| --- | --- | +| Scrape a URL to markdown, HTML, JSON or a screenshot | `client.web.scrape` | +| Crawl a site and get every page as markdown | `client.web.web_crawl_md` | +| Map every URL on a domain | `client.web.map_urls` | +| Search the web | `client.web.search` | +| Take a screenshot of a page | `client.web.screenshot` | +| Parse PDFs and documents | `client.parse.handle` | +| Run thousands of URLs as a batch | `client.batch.submit` | +| Watch a page for changes | `client.monitors.create` | +| Look up a company's logo, colors and brand data | `client.brand.retrieve` | + +## Use it from an AI agent + +Context.dev also ships as a plugin for [Claude](https://github.com/context-dot-dev/claude-plugin), [Cursor](https://github.com/context-dot-dev/cursor-plugin) and [Gemini CLI](https://github.com/context-dot-dev/gemini-cli-context), and as tools for [LangChain](https://github.com/context-dot-dev/langchain-context) and [Haystack](https://github.com/context-dot-dev/context-haystack). + While you can provide an `api_key` keyword argument, we recommend using [python-dotenv](https://pypi.org/project/python-dotenv/) to add `CONTEXT_DEV_API_KEY="My API Key"` to your `.env` file From a11a64a55740717c8dc63a1056a8c5f3a69f1159 Mon Sep 17 00:00:00 2001 From: Yahia Bakour Date: Thu, 8 Oct 2026 12:51:13 -0400 Subject: [PATCH 2/3] docs: remove Stainless README attribution --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 207f49a..daf64da 100644 --- a/README.md +++ b/README.md @@ -5,8 +5,6 @@ Context.dev is a web scraping API for AI agents and LLMs. This SDK turns any URL into clean, LLM-ready markdown, crawls whole sites, searches the web, takes screenshots and extracts structured JSON against a schema you define, all with one API key. Proxies, JavaScript rendering and anti-bot handling run on Context.dev's side, so there is no headless browser to host. -It is generated with [Stainless](https://www.stainless.com/). - ## Documentation The REST API documentation can be found on [docs.context.dev](https://docs.context.dev/). The full API of this library can be found in [api.md](api.md). From 78920ab980591450b5059e92fc4fa08291f2a05c Mon Sep 17 00:00:00 2001 From: Yahia Bakour Date: Thu, 8 Oct 2026 13:03:48 -0400 Subject: [PATCH 3/3] docs: demonstrate scraping formats throughout README --- README.md | 112 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index daf64da..334a777 100644 --- a/README.md +++ b/README.md @@ -18,21 +18,23 @@ pip install context.dev ## Usage +Set `CONTEXT_DEV_API_KEY` to your API key; the client reads it automatically. + +### Scrape markdown and HTML + The full API of this library can be found in [api.md](api.md). ```python -import os from context.dev import ContextDev -client = ContextDev( - api_key=os.environ.get("CONTEXT_DEV_API_KEY"), # This is the default and can be omitted -) +client = ContextDev() -brand = client.brand.retrieve( - domain="REPLACE_ME", - type="by_domain", +page = client.web.scrape( + url="https://example.com", + formats={"markdown": True, "html": True}, ) -print(brand.request_id) +print(page.markdown.data) +print(page.html.data) ``` ### Extract structured JSON @@ -60,11 +62,44 @@ page = client.web.scrape( print(page.json_.data) ``` +### Extract relevant highlights + +Return the passages that answer a question about the page. + +```python +from context.dev import ContextDev + +client = ContextDev() + +page = client.web.scrape( + url="https://example.com", + formats={"highlights": True}, + highlights_params={"query": "What is this domain used for?"}, +) +print(page.highlights.data) +``` + +### Take a screenshot + +The screenshot is returned as a base64 image data URL. + +```python +from context.dev import ContextDev + +client = ContextDev() + +page = client.web.scrape( + url="https://example.com", + formats={"screenshot": True}, +) +print(page.screenshot.data) +``` + ## What you can do | Task | Method | | --- | --- | -| Scrape a URL to markdown, HTML, JSON or a screenshot | `client.web.scrape` | +| Scrape a URL to markdown, HTML, JSON, highlights or a screenshot | `client.web.scrape` | | Crawl a site and get every page as markdown | `client.web.web_crawl_md` | | Map every URL on a domain | `client.web.map_urls` | | Search the web | `client.web.search` | @@ -98,11 +133,11 @@ client = AsyncContextDev( async def main() -> None: - brand = await client.brand.retrieve( - domain="REPLACE_ME", - type="by_domain", + page = await client.web.scrape( + url="https://example.com", + formats={"markdown": True}, ) - print(brand.request_id) + print(page.markdown.data) asyncio.run(main()) @@ -135,11 +170,11 @@ async def main() -> None: api_key=os.environ.get("CONTEXT_DEV_API_KEY"), # This is the default and can be omitted http_client=DefaultAioHttpClient(), ) as client: - brand = await client.brand.retrieve( - domain="REPLACE_ME", - type="by_domain", + page = await client.web.scrape( + url="https://example.com", + formats={"markdown": True}, ) - print(brand.request_id) + print(page.markdown.data) asyncio.run(main()) @@ -163,12 +198,12 @@ from context.dev import ContextDev client = ContextDev() -brand = client.brand.retrieve( - domain="xxx", - type="by_domain", +page = client.web.scrape( + url="https://example.com", + formats={"markdown": True}, timeout_opts={"milliseconds": 1000}, ) -print(brand.timeout_opts) +print(page.markdown.data) ``` ## Handling errors @@ -187,9 +222,9 @@ from context.dev import ContextDev client = ContextDev() try: - client.brand.retrieve( - domain="REPLACE_ME", - type="by_domain", + client.web.scrape( + url="https://example.com", + formats={"markdown": True}, ) except context.dev.APIConnectionError as e: print("The server could not be reached") @@ -233,9 +268,9 @@ client = ContextDev( ) # Or, configure per-request: -client.with_options(max_retries=5).brand.retrieve( - domain="REPLACE_ME", - type="by_domain", +client.with_options(max_retries=5).web.scrape( + url="https://example.com", + formats={"markdown": True}, ) ``` @@ -245,6 +280,7 @@ By default requests time out after 1 minute. You can configure this with a `time which accepts a float or an [`httpx.Timeout`](https://www.python-httpx.org/advanced/timeouts/#fine-tuning-the-configuration) object: ```python +import httpx from context.dev import ContextDev # Configure the default for all requests: @@ -259,9 +295,9 @@ client = ContextDev( ) # Override per-request: -client.with_options(timeout=5.0).brand.retrieve( - domain="REPLACE_ME", - type="by_domain", +client.with_options(timeout=5.0).web.scrape( + url="https://example.com", + formats={"markdown": True}, ) ``` @@ -303,14 +339,14 @@ The "raw" Response object can be accessed by prefixing `.with_raw_response.` to from context.dev import ContextDev client = ContextDev() -response = client.brand.with_raw_response.retrieve( - domain="REPLACE_ME", - type="by_domain", +response = client.web.with_raw_response.scrape( + url="https://example.com", + formats={"markdown": True}, ) print(response.headers.get('X-My-Header')) -brand = response.parse() # get the object that `brand.retrieve()` would have returned -print(brand.request_id) +page = response.parse() +print(page.markdown.data) ``` These methods return an [`APIResponse`](https://github.com/context-dot-dev/context-python-sdk/tree/main/src/context/dev/_response.py) object. @@ -324,9 +360,9 @@ The above interface eagerly reads the full response body when you make the reque To stream the response body, use `.with_streaming_response` instead, which requires a context manager and only reads the response body once you call `.read()`, `.text()`, `.json()`, `.iter_bytes()`, `.iter_text()`, `.iter_lines()` or `.parse()`. In the async client, these are async methods. ```python -with client.brand.with_streaming_response.retrieve( - domain="REPLACE_ME", - type="by_domain", +with client.web.with_streaming_response.scrape( + url="https://example.com", + formats={"markdown": True}, ) as response: print(response.headers.get("X-My-Header"))