diff --git a/.release-please-manifest.json b/.release-please-manifest.json index caf5ca3..59acac4 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.26.0" + ".": "0.27.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index af82291..4eb43d2 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ -configured_endpoints: 23 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-079e4949baaeab93254aa8a651f4e9976e95387b1633002e925114ce36f29013.yml -openapi_spec_hash: 331f27b62376aaf9c311af03a21d31b8 -config_hash: ada0970327c433e075b4ceef583e0755 +configured_endpoints: 24 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/context-dev/context.dev-cc8ca2c80e73bdcff491c96d9289c9e88d831215375ef83632cc51f8e8e06183.yml +openapi_spec_hash: bf55dce481efcf2f24bc9ff59c2e3f61 +config_hash: cf2c7f48b329e0a61612b30cd2755ad1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7167d47..1d73219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.27.0 (2026-05-31) + +Full Changelog: [v0.26.0...v0.27.0](https://github.com/context-dot-dev/context-python-sdk/compare/v0.26.0...v0.27.0) + +### Features + +* **api:** manual updates ([1ebc67e](https://github.com/context-dot-dev/context-python-sdk/commit/1ebc67ec3c939ad1becf27cf48780454ef60d583)) + ## 0.26.0 (2026-05-30) Full Changelog: [v0.25.0...v0.26.0](https://github.com/context-dot-dev/context-python-sdk/compare/v0.25.0...v0.26.0) diff --git a/README.md b/README.md index 1aa1c6f..366323c 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,17 @@ from context.dev import ContextDev client = ContextDev() -response = client.web.screenshot( - viewport={}, +response = client.web.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + pdf={}, ) -print(response.viewport) +print(response.pdf) ``` ## Handling errors diff --git a/api.md b/api.md index 04e3161..4fa444b 100644 --- a/api.md +++ b/api.md @@ -4,6 +4,7 @@ Types: ```python from context.dev.types import ( + WebExtractResponse, WebExtractFontsResponse, WebExtractStyleguideResponse, WebScreenshotResponse, @@ -18,6 +19,7 @@ from context.dev.types import ( Methods: +- client.web.extract(\*\*params) -> WebExtractResponse - client.web.extract_fonts(\*\*params) -> WebExtractFontsResponse - client.web.extract_styleguide(\*\*params) -> WebExtractStyleguideResponse - client.web.screenshot(\*\*params) -> WebScreenshotResponse diff --git a/pyproject.toml b/pyproject.toml index 1aabc9e..aced26b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "context.dev" -version = "0.26.0" +version = "0.27.0" description = "The official Python library for the context.dev API" dynamic = ["readme"] license = "Apache-2.0" diff --git a/src/context/dev/_version.py b/src/context/dev/_version.py index e7fc5fd..e1a2539 100644 --- a/src/context/dev/_version.py +++ b/src/context/dev/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "context.dev" -__version__ = "0.26.0" # x-release-please-version +__version__ = "0.27.0" # x-release-please-version diff --git a/src/context/dev/resources/web.py b/src/context/dev/resources/web.py index 32c357c..95d25b5 100644 --- a/src/context/dev/resources/web.py +++ b/src/context/dev/resources/web.py @@ -2,12 +2,14 @@ from __future__ import annotations +from typing import Dict from typing_extensions import Literal import httpx from ..types import ( web_search_params, + web_extract_params, web_screenshot_params, web_web_crawl_md_params, web_extract_fonts_params, @@ -29,6 +31,7 @@ ) from .._base_client import make_request_options from ..types.web_search_response import WebSearchResponse +from ..types.web_extract_response import WebExtractResponse from ..types.web_screenshot_response import WebScreenshotResponse from ..types.web_web_crawl_md_response import WebWebCrawlMdResponse from ..types.web_extract_fonts_response import WebExtractFontsResponse @@ -61,6 +64,99 @@ def with_streaming_response(self) -> WebResourceWithStreamingResponse: """ return WebResourceWithStreamingResponse(self) + def extract( + self, + *, + schema: Dict[str, object], + url: str, + fact_check: bool | Omit = omit, + follow_subdomains: bool | Omit = omit, + include_frames: bool | Omit = omit, + instructions: str | Omit = omit, + max_age_ms: int | Omit = omit, + pdf: web_extract_params.Pdf | Omit = omit, + stop_after_ms: int | Omit = omit, + timeout_ms: int | Omit = omit, + wait_for_ms: int | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> WebExtractResponse: + """ + Crawl a website, convert pages to Markdown using the scrape cache, and extract + structured data into the provided JSON Schema. The schema must describe the + response data object. This endpoint does not accept targeted page-type + selection. + + Args: + schema: JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + Schema generated from a Zod object; Python users can pass the equivalent JSON + Schema object. + + url: The starting website URL to crawl and extract from. Must include http:// or + https://. + + fact_check: When true (default), every returned value must be grounded in facts stated on + the page; fields that cannot be supported by the page are returned as + null/empty. When false, the model may make reasonable inferences and derivations + from the page content (e.g. ideal customer, competitor analysis, + recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + metrics) faithful to the source. + + follow_subdomains: When true, follow links on subdomains of the starting URL's domain. + + include_frames: When true, iframe contents are included in Markdown before extraction. + + instructions: Optional extraction guidance, such as which facts to prioritize or how to + interpret fields in the schema. + + max_age_ms: Return cached scrape results if a prior scrape for the same parameters is + younger than this many milliseconds. + + stop_after_ms: Soft time budget for the crawl in milliseconds. + + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer + than this value, it will be aborted with a 408 status code. Maximum allowed + value is 300000ms (5 minutes). + + wait_for_ms: Optional browser wait time in milliseconds after initial page load for each + crawled page. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return self._post( + "/web/extract", + body=maybe_transform( + { + "schema": schema, + "url": url, + "fact_check": fact_check, + "follow_subdomains": follow_subdomains, + "include_frames": include_frames, + "instructions": instructions, + "max_age_ms": max_age_ms, + "pdf": pdf, + "stop_after_ms": stop_after_ms, + "timeout_ms": timeout_ms, + "wait_for_ms": wait_for_ms, + }, + web_extract_params.WebExtractParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=WebExtractResponse, + ) + def extract_fonts( self, *, @@ -758,6 +854,99 @@ def with_streaming_response(self) -> AsyncWebResourceWithStreamingResponse: """ return AsyncWebResourceWithStreamingResponse(self) + async def extract( + self, + *, + schema: Dict[str, object], + url: str, + fact_check: bool | Omit = omit, + follow_subdomains: bool | Omit = omit, + include_frames: bool | Omit = omit, + instructions: str | Omit = omit, + max_age_ms: int | Omit = omit, + pdf: web_extract_params.Pdf | Omit = omit, + stop_after_ms: int | Omit = omit, + timeout_ms: int | Omit = omit, + wait_for_ms: int | Omit = omit, + # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. + # The extra values given here take precedence over values defined on the client or passed to this method. + extra_headers: Headers | None = None, + extra_query: Query | None = None, + extra_body: Body | None = None, + timeout: float | httpx.Timeout | None | NotGiven = not_given, + ) -> WebExtractResponse: + """ + Crawl a website, convert pages to Markdown using the scrape cache, and extract + structured data into the provided JSON Schema. The schema must describe the + response data object. This endpoint does not accept targeted page-type + selection. + + Args: + schema: JSON Schema for the returned data object. TypeScript Zod users can pass a JSON + Schema generated from a Zod object; Python users can pass the equivalent JSON + Schema object. + + url: The starting website URL to crawl and extract from. Must include http:// or + https://. + + fact_check: When true (default), every returned value must be grounded in facts stated on + the page; fields that cannot be supported by the page are returned as + null/empty. When false, the model may make reasonable inferences and derivations + from the page content (e.g. ideal customer, competitor analysis, + recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + metrics) faithful to the source. + + follow_subdomains: When true, follow links on subdomains of the starting URL's domain. + + include_frames: When true, iframe contents are included in Markdown before extraction. + + instructions: Optional extraction guidance, such as which facts to prioritize or how to + interpret fields in the schema. + + max_age_ms: Return cached scrape results if a prior scrape for the same parameters is + younger than this many milliseconds. + + stop_after_ms: Soft time budget for the crawl in milliseconds. + + timeout_ms: Optional timeout in milliseconds for the request. If the request takes longer + than this value, it will be aborted with a 408 status code. Maximum allowed + value is 300000ms (5 minutes). + + wait_for_ms: Optional browser wait time in milliseconds after initial page load for each + crawled page. + + extra_headers: Send extra headers + + extra_query: Add additional query parameters to the request + + extra_body: Add additional JSON properties to the request + + timeout: Override the client-level default timeout for this request, in seconds + """ + return await self._post( + "/web/extract", + body=await async_maybe_transform( + { + "schema": schema, + "url": url, + "fact_check": fact_check, + "follow_subdomains": follow_subdomains, + "include_frames": include_frames, + "instructions": instructions, + "max_age_ms": max_age_ms, + "pdf": pdf, + "stop_after_ms": stop_after_ms, + "timeout_ms": timeout_ms, + "wait_for_ms": wait_for_ms, + }, + web_extract_params.WebExtractParams, + ), + options=make_request_options( + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout + ), + cast_to=WebExtractResponse, + ) + async def extract_fonts( self, *, @@ -1439,6 +1628,9 @@ class WebResourceWithRawResponse: def __init__(self, web: WebResource) -> None: self._web = web + self.extract = to_raw_response_wrapper( + web.extract, + ) self.extract_fonts = to_raw_response_wrapper( web.extract_fonts, ) @@ -1472,6 +1664,9 @@ class AsyncWebResourceWithRawResponse: def __init__(self, web: AsyncWebResource) -> None: self._web = web + self.extract = async_to_raw_response_wrapper( + web.extract, + ) self.extract_fonts = async_to_raw_response_wrapper( web.extract_fonts, ) @@ -1505,6 +1700,9 @@ class WebResourceWithStreamingResponse: def __init__(self, web: WebResource) -> None: self._web = web + self.extract = to_streamed_response_wrapper( + web.extract, + ) self.extract_fonts = to_streamed_response_wrapper( web.extract_fonts, ) @@ -1538,6 +1736,9 @@ class AsyncWebResourceWithStreamingResponse: def __init__(self, web: AsyncWebResource) -> None: self._web = web + self.extract = async_to_streamed_response_wrapper( + web.extract, + ) self.extract_fonts = async_to_streamed_response_wrapper( web.extract_fonts, ) diff --git a/src/context/dev/types/__init__.py b/src/context/dev/types/__init__.py index 8c237b0..c24565a 100644 --- a/src/context/dev/types/__init__.py +++ b/src/context/dev/types/__init__.py @@ -4,8 +4,10 @@ from .web_search_params import WebSearchParams as WebSearchParams from .ai_ai_query_params import AIAIQueryParams as AIAIQueryParams +from .web_extract_params import WebExtractParams as WebExtractParams from .web_search_response import WebSearchResponse as WebSearchResponse from .ai_ai_query_response import AIAIQueryResponse as AIAIQueryResponse +from .web_extract_response import WebExtractResponse as WebExtractResponse from .brand_retrieve_params import BrandRetrieveParams as BrandRetrieveParams from .web_screenshot_params import WebScreenshotParams as WebScreenshotParams from .brand_retrieve_response import BrandRetrieveResponse as BrandRetrieveResponse diff --git a/src/context/dev/types/web_extract_params.py b/src/context/dev/types/web_extract_params.py new file mode 100644 index 0000000..cbf46e3 --- /dev/null +++ b/src/context/dev/types/web_extract_params.py @@ -0,0 +1,85 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Dict +from typing_extensions import Required, Annotated, TypedDict + +from .._utils import PropertyInfo + +__all__ = ["WebExtractParams", "Pdf"] + + +class WebExtractParams(TypedDict, total=False): + schema: Required[Dict[str, object]] + """JSON Schema for the returned data object. + + TypeScript Zod users can pass a JSON Schema generated from a Zod object; Python + users can pass the equivalent JSON Schema object. + """ + + url: Required[str] + """The starting website URL to crawl and extract from. + + Must include http:// or https://. + """ + + fact_check: Annotated[bool, PropertyInfo(alias="factCheck")] + """ + When true (default), every returned value must be grounded in facts stated on + the page; fields that cannot be supported by the page are returned as + null/empty. When false, the model may make reasonable inferences and derivations + from the page content (e.g. ideal customer, competitor analysis, + recommendations) while keeping verifiable specifics (names, quotes, URLs, dates, + metrics) faithful to the source. + """ + + follow_subdomains: Annotated[bool, PropertyInfo(alias="followSubdomains")] + """When true, follow links on subdomains of the starting URL's domain.""" + + include_frames: Annotated[bool, PropertyInfo(alias="includeFrames")] + """When true, iframe contents are included in Markdown before extraction.""" + + instructions: str + """ + Optional extraction guidance, such as which facts to prioritize or how to + interpret fields in the schema. + """ + + max_age_ms: Annotated[int, PropertyInfo(alias="maxAgeMs")] + """ + Return cached scrape results if a prior scrape for the same parameters is + younger than this many milliseconds. + """ + + pdf: Pdf + + stop_after_ms: Annotated[int, PropertyInfo(alias="stopAfterMs")] + """Soft time budget for the crawl in milliseconds.""" + + timeout_ms: Annotated[int, PropertyInfo(alias="timeoutMS")] + """Optional timeout in milliseconds for the request. + + If the request takes longer than this value, it will be aborted with a 408 + status code. Maximum allowed value is 300000ms (5 minutes). + """ + + wait_for_ms: Annotated[int, PropertyInfo(alias="waitForMs")] + """ + Optional browser wait time in milliseconds after initial page load for each + crawled page. + """ + + +class Pdf(TypedDict, total=False): + end: int + """Last 1-based PDF page to parse. + + Must be greater than or equal to start when both are provided. + """ + + should_parse: Annotated[bool, PropertyInfo(alias="shouldParse")] + """When true, PDF pages are fetched and parsed. When false, PDF pages are skipped.""" + + start: int + """First 1-based PDF page to parse.""" diff --git a/src/context/dev/types/web_extract_response.py b/src/context/dev/types/web_extract_response.py new file mode 100644 index 0000000..713d345 --- /dev/null +++ b/src/context/dev/types/web_extract_response.py @@ -0,0 +1,37 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List + +from pydantic import Field as FieldInfo + +from .._models import BaseModel + +__all__ = ["WebExtractResponse", "Metadata"] + + +class Metadata(BaseModel): + max_crawl_depth: int = FieldInfo(alias="maxCrawlDepth") + + num_failed: int = FieldInfo(alias="numFailed") + + num_skipped: int = FieldInfo(alias="numSkipped") + + num_succeeded: int = FieldInfo(alias="numSucceeded") + + num_urls: int = FieldInfo(alias="numUrls") + + +class WebExtractResponse(BaseModel): + data: Dict[str, object] + """Extracted data matching the request schema""" + + metadata: Metadata + + status: str + """Status of the response, e.g., 'ok'""" + + url: str + """The starting URL that was analyzed""" + + urls_analyzed: List[str] + """List of URLs whose Markdown was used for extraction""" diff --git a/tests/api_resources/test_web.py b/tests/api_resources/test_web.py index 46cbe44..1fab4ae 100644 --- a/tests/api_resources/test_web.py +++ b/tests/api_resources/test_web.py @@ -11,6 +11,7 @@ from tests.utils import assert_matches_type from context.dev.types import ( WebSearchResponse, + WebExtractResponse, WebScreenshotResponse, WebWebCrawlMdResponse, WebWebScrapeMdResponse, @@ -27,6 +28,85 @@ class TestWeb: parametrize = pytest.mark.parametrize("client", [False, True], indirect=True, ids=["loose", "strict"]) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_extract(self, client: ContextDev) -> None: + web = client.web.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_method_extract_with_all_params(self, client: ContextDev) -> None: + web = client.web.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + fact_check=True, + follow_subdomains=True, + include_frames=True, + instructions="instructions", + max_age_ms=0, + pdf={ + "end": 1, + "should_parse": True, + "start": 1, + }, + stop_after_ms=10000, + timeout_ms=1000, + wait_for_ms=0, + ) + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_raw_response_extract(self, client: ContextDev) -> None: + response = client.web.with_raw_response.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + web = response.parse() + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + def test_streaming_response_extract(self, client: ContextDev) -> None: + with client.web.with_streaming_response.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + web = response.parse() + assert_matches_type(WebExtractResponse, web, path=["response"]) + + assert cast(Any, response.is_closed) is True + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_extract_fonts(self, client: ContextDev) -> None: @@ -483,6 +563,85 @@ class TestAsyncWeb: "async_client", [False, True, {"http_client": "aiohttp"}], indirect=True, ids=["loose", "strict", "aiohttp"] ) + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_extract(self, async_client: AsyncContextDev) -> None: + web = await async_client.web.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_method_extract_with_all_params(self, async_client: AsyncContextDev) -> None: + web = await async_client.web.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + fact_check=True, + follow_subdomains=True, + include_frames=True, + instructions="instructions", + max_age_ms=0, + pdf={ + "end": 1, + "should_parse": True, + "start": 1, + }, + stop_after_ms=10000, + timeout_ms=1000, + wait_for_ms=0, + ) + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_raw_response_extract(self, async_client: AsyncContextDev) -> None: + response = await async_client.web.with_raw_response.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) + + assert response.is_closed is True + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + web = await response.parse() + assert_matches_type(WebExtractResponse, web, path=["response"]) + + @pytest.mark.skip(reason="Mock server tests are disabled") + @parametrize + async def test_streaming_response_extract(self, async_client: AsyncContextDev) -> None: + async with async_client.web.with_streaming_response.extract( + schema={ + "type": "bar", + "properties": "bar", + "required": "bar", + "additionalProperties": "bar", + }, + url="https://example.com", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + web = await response.parse() + assert_matches_type(WebExtractResponse, web, path=["response"]) + + assert cast(Any, response.is_closed) is True + @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_extract_fonts(self, async_client: AsyncContextDev) -> None: