From 3bb057b5e9fbdb744969e5e46f3f16a9d16cb01b Mon Sep 17 00:00:00 2001 From: "V. David Zvenyach" Date: Wed, 23 Sep 2026 08:22:24 -0500 Subject: [PATCH] fix(shapes): accept doc_role and the post-launch SLED fields the API serves Shapes naming `attachments(doc_role, doc_role_alt)` on opportunities and notices, or `delisted_at` / `meta(jurisdiction_declared)` on SLED solicitations, raised `ShapeValidationError` after the request had already been made, although the API returns all four. Re-vendors the filter/shape contract (Tango API 5.1.0) and regenerates the shape overlay from it. Re-vendoring alone did not fix opportunities: the overlay generator let the vehicles resource's `opportunity` embed, which exposes fewer attachment fields, replace the opportunity resource's own `attachments` schema. The generator now adds the missing fields instead of replacing the schema, and keeps every existing field's type. Adds observed types for the SLED resources, which the type probe predates, so the overlay stops guessing them from field names: the date fields parse to `datetime`, `category_codes` / `attachments` / `revisions` are lists, and the flags, sizes and counts are booleans and integers. Adds `delisted_at` to both SLED default shapes, matching the API's list default, and documents the new fields. Corrects the protest docstrings and API reference: three venues (GAO, COFC, SBA OHA), `naics_code` is the SBA OHA code at issue, and `case_number` examples cover each venue. Baselines `contract_appeals` and `ebuy/requests` as unmapped resources in the shape-coverage gate. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 13 + contracts/filter_shape_contract.json | 436 +++++++++++++++++- contracts/observed_shape_types.json | 582 +++++++++++++++++++++++++ contracts/shape_coverage_baseline.json | 4 +- docs/API_REFERENCE.md | 33 +- scripts/generate_shape_overlay.py | 21 +- tango/client.py | 6 +- tango/models.py | 30 +- tango/shapes/generated_overlay.py | 84 +++- tests/test_opportunity_attachments.py | 72 +++ tests/test_sled.py | 61 +++ 11 files changed, 1284 insertions(+), 58 deletions(-) create mode 100644 tests/test_opportunity_attachments.py diff --git a/CHANGELOG.md b/CHANGELOG.md index dcc18cb..df3b533 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **`attachments(doc_role, doc_role_alt)` on opportunities and notices no longer raises `ShapeValidationError`.** The Tango API serves both fields on a Pro plan or above, but the SDK rejected them client-side after the request had already been made. `doc_role` says what an attachment is for (`requirement`, `instructions`, `pricing`, `terms`, `reference` or `unknown`) and `doc_role_alt` is a runner-up role that is usually null. Both must be named in the shape, since `attachments(*)` does not include them, and an attachment that has not been classified omits both keys, so read them with `.get()`. +- **The shape overlay generator no longer lets one resource's view of a model hide fields from another's.** An opportunity is reachable both from `list_opportunities()` and as the `opportunity` expand on vehicles, and the vehicles embed exposes fewer attachment fields. Whichever tree was generated last replaced the other, so the opportunity schema lost fields its own endpoint serves. The generator now adds the missing fields instead of replacing the schema, and keeps every existing field's type as it was. +- **SLED solicitations accept `delisted_at` and `meta(jurisdiction_declared)`.** Both were rejected client-side even though the API returns them. `delisted_at` is when a portal dropped a solicitation from its listing before its deadline, which is what `status_reason="delisted"` means. `meta.jurisdiction_declared` says whether the source stated the jurisdiction level itself or Tango classified it from the issuer's name. `delisted_at` is now in both `SLED_OPPORTUNITIES_MINIMAL` and `SLED_OPPORTUNITIES_COMPREHENSIVE`, matching the API's own list default. + +### Changed +- **SLED response values now carry the types the API returns.** The SDK's SLED schema had guessed several types from field names. `posted_date`, `response_deadline`, `response_deadline_original` and `bid_opening_date` on `SledOpportunity` now parse to timezone-aware `datetime` values (before, `posted_date` and `bid_opening_date` were truncated to a `date` and the two deadlines stayed strings), as does `estimated_advertisement_date` on `SledForecast`. `estimated_value(min, max)` on forecasts now parses to `Decimal`. `category_codes` is typed as a list of `{scheme, code}` objects, `attachments` and `revisions` as lists, `meta(last_change_source_declared)` and `revisions(source_declared)` as booleans, and `attachments(size_bytes, pages)` and `revisions(sequence)` as integers. If you compared any of the date fields as strings, compare them as `datetime` values instead. +- Re-vendored `contracts/filter_shape_contract.json` (Tango API 5.1.0) and regenerated `tango/shapes/generated_overlay.py`. The two resources the SDK does not wrap yet, `contract_appeals` and `ebuy/requests`, are recorded in `contracts/shape_coverage_baseline.json` as known gaps. + +### Documentation +- `list_protests()` and `docs/API_REFERENCE.md` now describe all three protest venues: GAO, the U.S. Court of Federal Claims (COFC) and the SBA Office of Hearings and Appeals (SBA OHA). `naics_code` is documented as the NAICS code at issue in an SBA OHA size or NAICS appeal, which GAO and COFC protests never carry, rather than the solicitation's code. `case_number` examples now cover each venue's format, and `naics_code` is listed among the filter parameters. +- `docs/API_REFERENCE.md` documents the document-role fields on opportunity and notice attachments, and `delisted_at` and `meta.jurisdiction_declared` on SLED solicitations. + ## [1.6.0] - 2026-09-10 ### Added diff --git a/contracts/filter_shape_contract.json b/contracts/filter_shape_contract.json index 3fa44aa..0e47b14 100644 --- a/contracts/filter_shape_contract.json +++ b/contracts/filter_shape_contract.json @@ -1,6 +1,6 @@ { "meta": { - "api_version": "4.25.1", + "api_version": "5.1.0", "description": "Canonical API filter/shape contract. Downstream consumers (SDK, MCP) should validate their conformance against this manifest.", "generated_from": "scripts/filter_shape_conformance.py", "schema_version": 2 @@ -252,7 +252,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -994,7 +994,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -2471,6 +2471,156 @@ "show_shapes" ] }, + "contract_appeals": { + "basename": "contractappeal", + "docs_file": null, + "docs_params": [], + "prefix": "contract_appeals", + "resource_key": "contract_appeals", + "runtime": { + "filter_params": [ + "appellant", + "board", + "decision_date_after", + "decision_date_before", + "decision_type", + "docket", + "document_id", + "judge", + "listed", + "search" + ], + "filter_params_detail": { + "appellant": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "board": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "decision_date_after": { + "filter_class": "DateFilter", + "lookup": "gte", + "type": "date" + }, + "decision_date_before": { + "filter_class": "DateFilter", + "lookup": "lte", + "type": "date" + }, + "decision_type": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "docket": { + "filter_class": "DocketFilter", + "type": "string" + }, + "document_id": { + "filter_class": "CharFilter", + "type": "string" + }, + "judge": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "listed": { + "filter_class": "BooleanFilter", + "type": "boolean" + }, + "search": { + "filter_class": "ContractAppealSearchFilter", + "type": "string" + } + }, + "ordering_aliases": [], + "ordering_fields": [ + "appellant", + "decision_date", + "first_listed_at", + "rank" + ], + "pagination": { + "class": "StandardResultsSetPagination", + "max_page_size": 100 + }, + "shape": { + "expands": {}, + "fields": [ + "appellant", + "board", + "decision_date", + "decision_date_raw", + "decision_date_repaired", + "decision_text", + "decision_type", + "decision_type_raw", + "docket_numbers", + "docket_raw", + "docket_source", + "document_id", + "first_listed_at", + "judge", + "listed", + "listing_url", + "listing_year", + "text_char_count", + "text_status", + "url", + "uuid" + ] + }, + "shape_flat_paths": [ + "appellant", + "board", + "decision_date", + "decision_date_raw", + "decision_date_repaired", + "decision_text", + "decision_type", + "decision_type_raw", + "docket_numbers", + "docket_raw", + "docket_source", + "document_id", + "first_listed_at", + "judge", + "listed", + "listing_url", + "listing_year", + "text_char_count", + "text_status", + "url", + "uuid" + ], + "shape_gated_by_level": { + "business": [ + "decision_text" + ], + "free": [ + "decision_text" + ], + "pro": [ + "decision_text" + ] + }, + "shape_supported": true, + "shape_tier_required": "enterprise_a", + "viewset": "appeals.views.ContractAppealViewSet" + }, + "swagger_has_key": true, + "swagger_params": [ + "flat", + "flat_lists", + "joiner", + "limit", + "ordering", + "page", + "shape", + "show_shapes" + ] + }, "contracts": { "basename": "contract", "docs_file": null, @@ -2641,7 +2791,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -4016,6 +4166,241 @@ "swagger_has_key": false, "swagger_params": [] }, + "ebuy/requests": { + "basename": "ebuy-request", + "docs_file": null, + "docs_params": [], + "prefix": "ebuy/requests", + "resource_key": "ebuy/requests", + "runtime": { + "filter_params": [ + "buyer_agency", + "close_date_after", + "close_date_before", + "contract_number", + "issue_date_after", + "issue_date_before", + "reference_number", + "request_type", + "rfq_id", + "schedule", + "search", + "sin", + "status" + ], + "filter_params_detail": { + "buyer_agency": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "close_date_after": { + "filter_class": "DateFilter", + "lookup": "gte", + "type": "date" + }, + "close_date_before": { + "filter_class": "DateFilter", + "lookup": "date__lte", + "type": "date" + }, + "contract_number": { + "filter_class": "CharFilter", + "type": "string" + }, + "issue_date_after": { + "filter_class": "DateFilter", + "lookup": "gte", + "type": "date" + }, + "issue_date_before": { + "filter_class": "DateFilter", + "lookup": "date__lte", + "type": "date" + }, + "reference_number": { + "filter_class": "SolicitationNumberFilter", + "type": "string" + }, + "request_type": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "rfq_id": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "schedule": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "search": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "sin": { + "filter_class": "BaseSmartFilter", + "type": "string" + }, + "status": { + "filter_class": "BaseSmartFilter", + "type": "string" + } + }, + "ordering_aliases": [], + "ordering_fields": [ + "close_date", + "issue_date", + "last_seen", + "modified" + ], + "pagination": { + "class": "StandardResultsSetPagination", + "max_page_size": 100 + }, + "shape": { + "expands": { + "attachments": { + "expands": {}, + "fields": [ + "doc_name", + "doc_path", + "doc_seq_num", + "doc_session_date", + "doc_type", + "is_link" + ] + }, + "organization": { + "expands": {}, + "fields": [ + "agency_code", + "agency_name", + "department_code", + "department_name", + "office_code", + "office_name", + "organization_id" + ] + } + }, + "fields": [ + "addresses", + "amendment_count", + "amendments", + "attachment_count", + "award_method", + "buyer_agency", + "buyer_agency_code", + "buyer_email", + "buyer_name", + "buyer_user_id", + "cancel_date", + "close_date", + "commercial_type", + "contract_type", + "description", + "detail_fetched", + "first_seen", + "follow_on", + "issue_date", + "last_mod_date", + "last_seen", + "line_items", + "link_count", + "mod_version", + "oco_aac", + "oco_agency", + "oco_name", + "oco_phone", + "oco_title", + "ocs_aac", + "ocs_agency", + "ocs_name", + "ocs_phone", + "ocs_title", + "pop_end_date", + "pop_start_date", + "qa_document_count", + "reference_number", + "request_type", + "rfq_id", + "schedule", + "sin", + "source_sought", + "status", + "title" + ] + }, + "shape_flat_paths": [ + "addresses", + "amendment_count", + "amendments", + "attachment_count", + "attachments", + "attachments.doc_name", + "attachments.doc_path", + "attachments.doc_seq_num", + "attachments.doc_session_date", + "attachments.doc_type", + "attachments.is_link", + "award_method", + "buyer_agency", + "buyer_agency_code", + "buyer_email", + "buyer_name", + "buyer_user_id", + "cancel_date", + "close_date", + "commercial_type", + "contract_type", + "description", + "detail_fetched", + "first_seen", + "follow_on", + "issue_date", + "last_mod_date", + "last_seen", + "line_items", + "link_count", + "mod_version", + "oco_aac", + "oco_agency", + "oco_name", + "oco_phone", + "oco_title", + "ocs_aac", + "ocs_agency", + "ocs_name", + "ocs_phone", + "ocs_title", + "organization", + "organization.agency_code", + "organization.agency_name", + "organization.department_code", + "organization.department_name", + "organization.office_code", + "organization.office_name", + "organization.organization_id", + "pop_end_date", + "pop_start_date", + "qa_document_count", + "reference_number", + "request_type", + "rfq_id", + "schedule", + "sin", + "source_sought", + "status", + "title" + ], + "shape_gated_by_level": {}, + "shape_supported": true, + "shape_tier_required": null, + "viewset": "ebuy.views.EbuyRequestViewSet" + }, + "swagger_has_key": false, + "swagger_params": [] + }, "entities": { "basename": "entity", "docs_file": null, @@ -4636,7 +5021,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -5368,7 +5753,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -6134,7 +6519,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -6477,7 +6862,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -8068,7 +8453,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -8708,7 +9093,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -9440,7 +9825,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -10692,6 +11077,8 @@ "expands": {}, "fields": [ "attachment_id", + "doc_role", + "doc_role_alt", "extracted_text", "file_size", "mime_type", @@ -10814,6 +11201,8 @@ "attachment_count", "attachments", "attachments.attachment_id", + "attachments.doc_role", + "attachments.doc_role_alt", "attachments.extracted_text", "attachments.file_size", "attachments.mime_type", @@ -10876,6 +11265,8 @@ ], "shape_gated_by_level": { "free": [ + "attachments.doc_role", + "attachments.doc_role_alt", "attachments.extracted_text" ] }, @@ -11123,6 +11514,8 @@ "expands": {}, "fields": [ "attachment_id", + "doc_role", + "doc_role_alt", "extracted_text", "file_size", "mime_type", @@ -11274,6 +11667,8 @@ "archive_date", "attachments", "attachments.attachment_id", + "attachments.doc_role", + "attachments.doc_role_alt", "attachments.extracted_text", "attachments.file_size", "attachments.mime_type", @@ -11357,6 +11752,8 @@ ], "shape_gated_by_level": { "free": [ + "attachments.doc_role", + "attachments.doc_role_alt", "attachments.extracted_text" ] }, @@ -11790,7 +12187,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -12141,7 +12538,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -12471,7 +12868,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -13885,6 +14282,7 @@ "expands": {}, "fields": [ "attachment_count", + "jurisdiction_declared", "last_change_source_declared", "last_revision_kind", "revision_count" @@ -13921,6 +14319,7 @@ "bid_opening_date", "bid_opening_raw", "category_codes", + "delisted_at", "description", "first_seen_at", "has_documents", @@ -13969,6 +14368,7 @@ "contact.email", "contact.name", "contact.phone", + "delisted_at", "description", "first_seen_at", "has_documents", @@ -13977,6 +14377,7 @@ "last_seen_at", "meta", "meta.attachment_count", + "meta.jurisdiction_declared", "meta.last_change_source_declared", "meta.last_revision_kind", "meta.revision_count", @@ -15075,7 +15476,6 @@ "expands": {}, "fields": [ "attachment_id", - "extracted_text", "file_size", "mime_type", "name", @@ -15681,7 +16081,6 @@ "opportunity.archive_date", "opportunity.attachments", "opportunity.attachments.attachment_id", - "opportunity.attachments.extracted_text", "opportunity.attachments.file_size", "opportunity.attachments.mime_type", "opportunity.attachments.name", @@ -15996,7 +16395,7 @@ "type": "string" }, "uei": { - "filter_class": "UppercaseCharFilter", + "filter_class": "UeiToEntityUuidFilter", "type": "string" } }, @@ -16577,12 +16976,14 @@ "assistance_listings", "budget/accounts", "business_types", + "contract_appeals", "contracts", "contracts_subawards", "departments", "dibbs/awards", "dibbs/rfps", "dibbs/rfqs", + "ebuy/requests", "entities", "entities_contracts", "entities_idvs", @@ -16628,6 +17029,7 @@ "dibbs/awards", "dibbs/rfps", "dibbs/rfqs", + "ebuy/requests", "entities_contracts", "entities_idvs", "entities_lcats", diff --git a/contracts/observed_shape_types.json b/contracts/observed_shape_types.json index aa4d965..ba69aef 100644 --- a/contracts/observed_shape_types.json +++ b/contracts/observed_shape_types.json @@ -11278,6 +11278,588 @@ }, "records_seen": 160 }, + "sled/forecasts": { + "paths": { + "agency": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contact": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "contact.email": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contact.name": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contact.phone": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contract_number": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contract_term": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "delivery_location": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "description": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "estimated_advertisement_date": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "estimated_advertisement_raw": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "estimated_value": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "estimated_value.max": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "Decimal" + }, + "estimated_value.min": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "Decimal" + }, + "estimated_value.raw": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "first_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "forecast_id": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "has_documents": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "incumbent_name": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "last_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "mbe_dbe_goal": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "modified": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "organization": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "organization.agency": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "organization.level": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "organization.state": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "procurement_category": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "procurement_method": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "raw": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "source_status": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "source_url": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "state": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "title": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + } + }, + "records_seen": 200 + }, + "sled/opportunities": { + "paths": { + "agency": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments": { + "is_list": true, + "is_optional": false, + "kind": "object" + }, + "attachments.char_count": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "attachments.checksum": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.download_status": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.extension": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.extracted_text": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.extraction_status": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.first_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "attachments.is_contested": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "attachments.is_generated_summary": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "attachments.mime_type": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.name": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "attachments.pages": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "attachments.size_bytes": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "attachments.word_count": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "bid_opening_date": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "bid_opening_raw": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "category_codes": { + "is_list": true, + "is_optional": true, + "kind": "scalar", + "type": "dict" + }, + "contact": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "contact.email": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contact.name": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "contact.phone": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "delisted_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "description": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "first_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "has_documents": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "jurisdiction": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "last_change_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "last_seen_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "meta": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "meta.attachment_count": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "meta.jurisdiction_declared": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "meta.last_change_source_declared": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "meta.last_revision_kind": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "meta.revision_count": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "modified": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "opportunity_id": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "organization": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "organization.agency": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "organization.level": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "organization.state": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "posted_date": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "raw": { + "is_list": false, + "is_optional": false, + "kind": "object" + }, + "response_deadline": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "response_deadline_original": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "revisions": { + "is_list": true, + "is_optional": false, + "kind": "object" + }, + "revisions.changed_fields": { + "is_list": true, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "revisions.changes": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "dict" + }, + "revisions.kind": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "revisions.observed_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "revisions.sequence": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "int" + }, + "revisions.source_declared": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "bool" + }, + "snippet": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "solicitation_number": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "solicitation_type": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "solicitation_type_source": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "source_status": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "source_url": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "state": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "status": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "status_computed_at": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "datetime" + }, + "status_reason": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + }, + "title": { + "is_list": false, + "is_optional": true, + "kind": "scalar", + "type": "str" + } + }, + "records_seen": 280 + }, "subawards": { "paths": { "award_key": { diff --git a/contracts/shape_coverage_baseline.json b/contracts/shape_coverage_baseline.json index 15207af..14bb89e 100644 --- a/contracts/shape_coverage_baseline.json +++ b/contracts/shape_coverage_baseline.json @@ -1,10 +1,12 @@ { "description": "Known reverse shape-coverage gaps (Tango exposes, SDK schema lacks), accepted as a tracked backlog. check_shape_coverage.py fails only on gaps NOT listed here. Burn down and regenerate with --update-baseline.", - "count": 14, + "count": 16, "known_gaps": [ "unmapped_resource|agencies_contracts_awarding|(root)|None", "unmapped_resource|agencies_contracts_funding|(root)|None", + "unmapped_resource|contract_appeals|(root)|None", "unmapped_resource|contracts_subawards|(root)|None", + "unmapped_resource|ebuy/requests|(root)|None", "unmapped_resource|entities_contracts|(root)|None", "unmapped_resource|entities_idvs|(root)|None", "unmapped_resource|entities_lcats|(root)|None", diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 20a26fc..03df19f 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -925,6 +925,25 @@ for opp in opportunities.results: - `naics_code` - Industry code - `psc_code` - Product/service code +**Document roles — `attachments(doc_role, doc_role_alt)`:** + +On a **Pro plan or above**, each attachment can say what the document is for. +`doc_role` is one of `requirement`, `instructions`, `pricing`, `terms`, `reference` or `unknown`; `doc_role_alt` is a runner-up role, and is null on nearly every attachment. + +```python +opp = client.get_opportunity( + opportunity_id, + shape="opportunity_id,title,attachments(name,url,doc_role,doc_role_alt)", +) +for attachment in opp["attachments"]: + print(attachment["name"], attachment.get("doc_role")) +``` + +- **You have to name them.** `attachments(*)` does not include either field, and no default shape does. +- **The keys are absent, not null**, on an attachment Tango has not classified. Use `attachment.get("doc_role")`. +- A Free-plan request that names them gets the response without them, plus an entry in `meta.upgrade_hints`. +- There is no filter on document role. + --- ## Notices @@ -1143,7 +1162,7 @@ contract = client.get_gsa_elibrary_contract("UUID_HERE") ## Protests -Bid protest records (GAO, COFC, etc.). +Bid protest records from three venues: GAO, the U.S. Court of Federal Claims (COFC), and the SBA Office of Hearings and Appeals (SBA OHA). ### list_protests() @@ -1166,20 +1185,22 @@ protests = client.list_protests( filed_date_before=None, decision_date_after=None, decision_date_before=None, + naics_code=None, search=None, ) ``` **Filter Parameters:** -- `source_system` - Filter by source system (e.g., `"gao"`) +- `source_system` - Filter by source system: `"gao"`, `"cofc"`, or `"sba_oha"` - `outcome` - Filter by outcome (e.g., `"Denied"`, `"Dismissed"`, `"Withdrawn"`, `"Sustained"`) - `case_type` - Filter by case type - `agency` - Filter by protested agency -- `case_number` - Filter by case number (e.g., `"b-423274"`) +- `case_number` - Filter by base case number, matched case-insensitively (e.g., `"b-423274"` for GAO, `"26-1391"` for COFC, `"SIZ-6100"` for SBA OHA) - `solicitation_number` - Filter by solicitation number - `protester` - Search by protester name - `filed_date_after` / `filed_date_before` - Filed date range - `decision_date_after` / `decision_date_before` - Decision date range +- `naics_code` - The NAICS code at issue in an SBA OHA size or NAICS appeal (e.g., `"541519"`). GAO and COFC protests carry no NAICS code, so this filter returns SBA OHA records only - `search` - Full-text search **Returns:** [PaginatedResponse](#paginatedresponse) with protest dictionaries @@ -1323,6 +1344,8 @@ row = client.get_sled_opportunity( - `meta.attachment_count` can be **lower** than `len(row["attachments"])`. Some portals auto-generate a cover sheet alongside the real documents; it is listed and flagged `is_generated_summary`, but excluded from the count and from `has_documents`. The count answers "does this hold its solicitation package"; the array answers "what files exist". - `size_bytes` and `char_count` only mean something as a pair — 3 MB that yielded no characters is a scan awaiting OCR. - `raw(*)` needs a Small plan or above, and is explicitly unstable: its shape varies by portal platform. +- `delisted_at` is when a portal dropped the solicitation from its listing before its deadline, and is what `status_reason="delisted"` means. It is null when the solicitation was never delisted or has been seen again since. +- `meta.jurisdiction_declared` says whether the source stated the jurisdiction level itself. `False` means Tango classified it from the issuer's name, which is the common case on a state's central portal. **Reading a document body — `attachments(extracted_text)`:** @@ -2210,8 +2233,8 @@ entity = client.get_entity("UEI_KEY", shape=ShapeConfig.ENTITIES_COMPREHENSIVE) | `VEHICLE_ORDERS_MINIMAL` | `list_vehicle_orders` | key, piid, award_date, recipient(display_name,uei), total_contract_value, obligated | | `ITDASHBOARD_INVESTMENTS_MINIMAL` | `list_itdashboard_investments` | Minimal IT Dashboard investment fields | | `ITDASHBOARD_INVESTMENTS_COMPREHENSIVE` | `get_itdashboard_investment` | Full investment fields: uii, agency_code, agency_name, bureau_code, bureau_name, investment_title, type_of_investment, part_of_it_portfolio, updated_time, url | -| `SLED_OPPORTUNITIES_MINIMAL` | `list_sled_opportunities` | opportunity_id, solicitation_number, solicitation_type, title, state, jurisdiction, agency, status, status_reason, posted_date, response_deadline, source_url, has_documents, first_seen_at, last_change_seen_at (no `description` — detail-only on the API) | -| `SLED_OPPORTUNITIES_COMPREHENSIVE` | `get_sled_opportunity` | Full solicitation with description, the raw portal status, both deadlines, bid opening, category codes, and organization / contact / meta / attachments / revisions | +| `SLED_OPPORTUNITIES_MINIMAL` | `list_sled_opportunities` | opportunity_id, solicitation_number, solicitation_type, title, state, jurisdiction, agency, status, status_reason, delisted_at, posted_date, response_deadline, source_url, has_documents, first_seen_at, last_change_seen_at (no `description` — detail-only on the API) | +| `SLED_OPPORTUNITIES_COMPREHENSIVE` | `get_sled_opportunity` | Full solicitation with description, the raw portal status, the delisting timestamp, both deadlines, bid opening, category codes, and organization / contact / meta / attachments / revisions | | `SLED_REVISIONS_MINIMAL` | `list_sled_opportunity_revisions` | observed_at, sequence, kind, changed_fields, source_declared (omits `changes`, which needs a Small plan) | | `SLED_FORECASTS_MINIMAL` | `list_sled_forecasts` | forecast_id, state, agency, title, estimated_advertisement_date, estimated_advertisement_raw, procurement_category, procurement_method, contract_number, incumbent_name, source_url, estimated_value(*) | | `SLED_FORECASTS_COMPREHENSIVE` | `get_sled_forecast` | Full forecast with description, contract_term, mbe_dbe_goal, delivery_location, and organization / contact / estimated_value | diff --git a/scripts/generate_shape_overlay.py b/scripts/generate_shape_overlay.py index a9022fd..28f8f5a 100644 --- a/scripts/generate_shape_overlay.py +++ b/scripts/generate_shape_overlay.py @@ -174,6 +174,20 @@ def expand_entry(res: str, path: str, ename: str, enode: dict) -> dict: report_rows: list[str] = [] +def put(container: str, name: str, e: dict) -> None: + """Record an overlay entry, unioning nested schemas when two resources reach the same container. + + A model can be reached from its own resource and as another resource's expand (`opportunities` vs `vehicles.opportunity`), and those trees can expose different sub-fields. + Overwriting let a narrower embed hide fields the resource itself serves; the union only adds the missing ones, and the later entry still decides every field both trees define, so no existing type moves. + """ + cur = overlay.setdefault(container, {}).get(name) + nested = {cur.get("nested") if cur else None, e.get("nested")} + if cur and None not in nested and "CodeDescription" not in nested and len(nested) == 2: + merged = {**nested_schemas[cur["nested"]], **nested_schemas[e["nested"]]} + e = {**e, "nested": intern_nested(re.sub(r"\d+$", "", e["nested"]), merged)} + overlay[container][name] = e + + def walk(res: str, path: str, node: dict, schema, container: str) -> None: if schema is None: return @@ -182,7 +196,7 @@ def walk(res: str, path: str, node: dict, schema, container: str) -> None: for f in fields: if f != "*" and f not in schema: t, lst = resolve_scalar(res, path, f) - overlay.setdefault(container, {})[f] = entry(t, True, lst) + put(container, f, entry(t, True, lst)) report_rows.append(f"{res}:{path or '(root)'}.{f} -> {t}{'[]' if lst else ''}") for ename, enode in (node.get("expands") or {}).items(): fs = schema.get(ename) @@ -192,7 +206,7 @@ def walk(res: str, path: str, node: dict, schema, container: str) -> None: if fs is None or child_schema is None: if is_wildcard(enode) and fs is not None: continue - overlay.setdefault(container, {})[ename] = expand_entry(res, path, ename, enode) + put(container, ename, expand_entry(res, path, ename, enode)) report_rows.append(f"{res}:{path or '(root)'}.{ename} -> expand") else: walk(res, child_path, enode, child_schema, container=nested_name) @@ -214,7 +228,8 @@ def walk(res: str, path: str, node: dict, schema, container: str) -> None: for cname, cnode in (shape.get("expands") or {}).items(): s[cname] = expand_entry(rkey, "", cname, cnode) if model_name: - overlay[model_name] = {**overlay.get(model_name, {}), **s} + for name, e in s.items(): + put(model_name, name, e) continue walk(rkey, "", shape, schema, container=model_name) diff --git a/tango/client.py b/tango/client.py index 0db44c8..26c8d3e 100644 --- a/tango/client.py +++ b/tango/client.py @@ -2592,18 +2592,18 @@ def list_protests( shape: Response shape string (defaults to minimal shape) flat: If True, flatten nested objects in shaped response flat_lists: If True, flatten arrays using indexed keys - source_system: Filter by source system (e.g. gao) + source_system: Filter by source system: gao, cofc, or sba_oha outcome: Filter by outcome (e.g. Denied, Dismissed, Withdrawn, Sustained) case_type: Filter by case type agency: Filter by protested agency text - case_number: Filter by case number (e.g. b-423274) + case_number: Filter by base case number, matched case-insensitively (e.g. b-423274 for GAO, 26-1391 for COFC, SIZ-6100 for SBA OHA) solicitation_number: Filter by exact solicitation number protester: Filter by protester name text filed_date_after: Filed date on or after filed_date_before: Filed date on or before decision_date_after: Decision date on or after decision_date_before: Decision date on or before - naics_code: Filter by the solicitation's NAICS code (e.g. 541519) + naics_code: Filter by the NAICS code at issue in an SBA OHA size or NAICS appeal (e.g. 541519); GAO and COFC protests carry no NAICS code and never match search: Full-text search over protest searchable fields """ params: dict[str, Any] = {"page": page, "limit": min(limit, 100)} diff --git a/tango/models.py b/tango/models.py index dbf5944..cb4dd26 100644 --- a/tango/models.py +++ b/tango/models.py @@ -559,7 +559,13 @@ class Forecast: @dataclass class Opportunity: - """Schema definition for Opportunity (not used for instances)""" + """Schema definition for Opportunity (not used for instances) + + The ``attachments(...)`` expand can carry two document-role fields on a Pro plan or above: ``doc_role``, what the document is for (one of ``requirement``, ``instructions``, ``pricing``, ``terms``, ``reference``, ``unknown``), and ``doc_role_alt``, a runner-up role that is usually null. + Both must be named, e.g. ``attachments(name,url,doc_role,doc_role_alt)``, because ``attachments(*)`` does not include them. + An attachment Tango has not classified omits both keys rather than returning null, so read them with ``.get()``. + A Free-plan request that names them gets the response without them, plus an entry in ``meta.upgrade_hints``. + """ opportunity_id: str title: str @@ -573,7 +579,13 @@ class Opportunity: @dataclass class Notice: - """Schema definition for Notice (not used for instances)""" + """Schema definition for Notice (not used for instances) + + The ``attachments(...)`` expand can carry two document-role fields on a Pro plan or above: ``doc_role``, what the document is for (one of ``requirement``, ``instructions``, ``pricing``, ``terms``, ``reference``, ``unknown``), and ``doc_role_alt``, a runner-up role that is usually null. + Both must be named, e.g. ``attachments(name,url,doc_role,doc_role_alt)``, because ``attachments(*)`` does not include them. + An attachment Tango has not classified omits both keys rather than returning null, so read them with ``.get()``. + A Free-plan request that names them gets the response without them, plus an entry in ``meta.upgrade_hints``. + """ notice_id: str title: str @@ -1050,6 +1062,10 @@ class SledOpportunity: Searching document text and reading it are separate: ``search=`` matches inside attachment text on every plan and returns no fragment of it, while ``attachments(extracted_text)`` serves the body on a Small plan or above. + + ``delisted_at`` is when a portal dropped the solicitation from its listing before its deadline, which is the reading behind ``status_reason="delisted"``; it is null when the solicitation was never delisted or has been seen again since. + + ``meta(jurisdiction_declared)`` is a boolean saying whether the source stated the jurisdiction level itself; ``False`` means Tango classified it from the issuer's name. """ opportunity_id: str | None = None @@ -1063,6 +1079,7 @@ class SledOpportunity: agency: str | None = None status: str | None = None status_reason: str | None = None + delisted_at: str | None = None status_computed_at: str | None = None source_status: str | None = None source_url: str | None = None @@ -1356,8 +1373,9 @@ class ShapeConfig: # API (median ~550 chars, tail past 120k), so it is not in the list default. SLED_OPPORTUNITIES_MINIMAL: Final = ( "opportunity_id,solicitation_number,solicitation_type,title,state," - "jurisdiction,agency,status,status_reason,posted_date,response_deadline," - "source_url,has_documents,first_seen_at,last_change_seen_at" + "jurisdiction,agency,status,status_reason,delisted_at,posted_date," + "response_deadline,source_url,has_documents,first_seen_at," + "last_change_seen_at" ) # Default for get_sled_opportunity(). `attachments(*)` deliberately does not @@ -1367,8 +1385,8 @@ class ShapeConfig: SLED_OPPORTUNITIES_COMPREHENSIVE: Final = ( "opportunity_id,solicitation_number,solicitation_type," "solicitation_type_source,title,description,state,jurisdiction,agency," - "status,status_reason,status_computed_at,source_status,source_url," - "posted_date,response_deadline,response_deadline_original," + "status,status_reason,delisted_at,status_computed_at,source_status," + "source_url,posted_date,response_deadline,response_deadline_original," "bid_opening_date,bid_opening_raw,category_codes,has_documents," "first_seen_at,last_seen_at,last_change_seen_at," "organization(*),contact(*),meta(*),attachments(*),revisions(*)" diff --git a/tango/shapes/generated_overlay.py b/tango/shapes/generated_overlay.py index 3911406..ac01621 100644 --- a/tango/shapes/generated_overlay.py +++ b/tango/shapes/generated_overlay.py @@ -81,7 +81,20 @@ } ATTACHMENTS_SCHEMA: dict[str, FieldSchema] = { + "doc_name": FieldSchema(name="doc_name", type=str, is_optional=True, is_list=False), + "doc_path": FieldSchema(name="doc_path", type=str, is_optional=True, is_list=False), + "doc_seq_num": FieldSchema(name="doc_seq_num", type=str, is_optional=True, is_list=False), + "doc_session_date": FieldSchema( + name="doc_session_date", type=date, is_optional=True, is_list=False + ), + "doc_type": FieldSchema(name="doc_type", type=str, is_optional=True, is_list=False), + "is_link": FieldSchema(name="is_link", type=bool, is_optional=True, is_list=False), +} + +ATTACHMENTS2_SCHEMA: dict[str, FieldSchema] = { "attachment_id": FieldSchema(name="attachment_id", type=str, is_optional=True, is_list=False), + "doc_role": FieldSchema(name="doc_role", type=str, is_optional=True, is_list=False), + "doc_role_alt": FieldSchema(name="doc_role_alt", type=str, is_optional=True, is_list=False), "extracted_text": FieldSchema(name="extracted_text", type=str, is_optional=True, is_list=False), "file_size": FieldSchema(name="file_size", type=int, is_optional=True, is_list=False), "mime_type": FieldSchema(name="mime_type", type=str, is_optional=True, is_list=False), @@ -92,8 +105,10 @@ "url": FieldSchema(name="url", type=str, is_optional=True, is_list=False), } -ATTACHMENTS2_SCHEMA: dict[str, FieldSchema] = { +ATTACHMENTS3_SCHEMA: dict[str, FieldSchema] = { "attachment_id": FieldSchema(name="attachment_id", type=str, is_optional=True, is_list=False), + "doc_role": FieldSchema(name="doc_role", type=str, is_optional=True, is_list=False), + "doc_role_alt": FieldSchema(name="doc_role_alt", type=str, is_optional=True, is_list=False), "extracted_text": FieldSchema(name="extracted_text", type=str, is_optional=True, is_list=False), "file_size": FieldSchema(name="file_size", type=int, is_optional=True, is_list=False), "mime_type": FieldSchema(name="mime_type", type=str, is_optional=True, is_list=False), @@ -104,7 +119,7 @@ "url": FieldSchema(name="url", type=str, is_optional=True, is_list=False), } -ATTACHMENTS3_SCHEMA: dict[str, FieldSchema] = { +ATTACHMENTS4_SCHEMA: dict[str, FieldSchema] = { "char_count": FieldSchema(name="char_count", type=int, is_optional=True, is_list=False), "checksum": FieldSchema(name="checksum", type=str, is_optional=True, is_list=False), "download_status": FieldSchema( @@ -124,13 +139,26 @@ ), "mime_type": FieldSchema(name="mime_type", type=str, is_optional=True, is_list=False), "name": FieldSchema(name="name", type=str, is_optional=True, is_list=False), - "pages": FieldSchema(name="pages", type=str, is_optional=True, is_list=False), - "size_bytes": FieldSchema(name="size_bytes", type=str, is_optional=True, is_list=False), + "pages": FieldSchema(name="pages", type=int, is_optional=True, is_list=False), + "size_bytes": FieldSchema(name="size_bytes", type=int, is_optional=True, is_list=False), "word_count": FieldSchema(name="word_count", type=int, is_optional=True, is_list=False), } -ATTACHMENTS4_SCHEMA: dict[str, FieldSchema] = { +ATTACHMENTS5_SCHEMA: dict[str, FieldSchema] = { "attachment_id": FieldSchema(name="attachment_id", type=str, is_optional=True, is_list=False), + "file_size": FieldSchema(name="file_size", type=str, is_optional=True, is_list=False), + "mime_type": FieldSchema(name="mime_type", type=str, is_optional=True, is_list=False), + "name": FieldSchema(name="name", type=str, is_optional=True, is_list=False), + "posted_date": FieldSchema(name="posted_date", type=date, is_optional=True, is_list=False), + "resource_id": FieldSchema(name="resource_id", type=str, is_optional=True, is_list=False), + "type": FieldSchema(name="type", type=str, is_optional=True, is_list=False), + "url": FieldSchema(name="url", type=str, is_optional=True, is_list=False), +} + +ATTACHMENTS6_SCHEMA: dict[str, FieldSchema] = { + "attachment_id": FieldSchema(name="attachment_id", type=str, is_optional=True, is_list=False), + "doc_role": FieldSchema(name="doc_role", type=str, is_optional=True, is_list=False), + "doc_role_alt": FieldSchema(name="doc_role_alt", type=str, is_optional=True, is_list=False), "extracted_text": FieldSchema(name="extracted_text", type=str, is_optional=True, is_list=False), "file_size": FieldSchema(name="file_size", type=str, is_optional=True, is_list=False), "mime_type": FieldSchema(name="mime_type", type=str, is_optional=True, is_list=False), @@ -375,8 +403,8 @@ } ESTIMATED_VALUE_SCHEMA: dict[str, FieldSchema] = { - "max": FieldSchema(name="max", type=str, is_optional=True, is_list=False), - "min": FieldSchema(name="min", type=str, is_optional=True, is_list=False), + "max": FieldSchema(name="max", type=Decimal, is_optional=True, is_list=False), + "min": FieldSchema(name="min", type=Decimal, is_optional=True, is_list=False), "raw": FieldSchema(name="raw", type=str, is_optional=True, is_list=False), } @@ -624,8 +652,11 @@ "attachment_count": FieldSchema( name="attachment_count", type=int, is_optional=True, is_list=False ), + "jurisdiction_declared": FieldSchema( + name="jurisdiction_declared", type=bool, is_optional=True, is_list=False + ), "last_change_source_declared": FieldSchema( - name="last_change_source_declared", type=str, is_optional=True, is_list=False + name="last_change_source_declared", type=bool, is_optional=True, is_list=False ), "last_revision_kind": FieldSchema( name="last_revision_kind", type=str, is_optional=True, is_list=False @@ -942,13 +973,13 @@ } REVISIONS_SCHEMA: dict[str, FieldSchema] = { - "changed_fields": FieldSchema(name="changed_fields", type=str, is_optional=True, is_list=False), - "changes": FieldSchema(name="changes", type=str, is_optional=True, is_list=False), + "changed_fields": FieldSchema(name="changed_fields", type=str, is_optional=True, is_list=True), + "changes": FieldSchema(name="changes", type=dict, is_optional=True, is_list=False), "kind": FieldSchema(name="kind", type=str, is_optional=True, is_list=False), "observed_at": FieldSchema(name="observed_at", type=datetime, is_optional=True, is_list=False), - "sequence": FieldSchema(name="sequence", type=str, is_optional=True, is_list=False), + "sequence": FieldSchema(name="sequence", type=int, is_optional=True, is_list=False), "source_declared": FieldSchema( - name="source_declared", type=str, is_optional=True, is_list=False + name="source_declared", type=bool, is_optional=True, is_list=False ), } @@ -1087,6 +1118,8 @@ "Attachments2": ATTACHMENTS2_SCHEMA, "Attachments3": ATTACHMENTS3_SCHEMA, "Attachments4": ATTACHMENTS4_SCHEMA, + "Attachments5": ATTACHMENTS5_SCHEMA, + "Attachments6": ATTACHMENTS6_SCHEMA, "Awardee": AWARDEE_SCHEMA, "AwardingOffice": AWARDING_OFFICE_SCHEMA, "AwardingOffice2": AWARDING_OFFICE2_SCHEMA, @@ -2185,7 +2218,7 @@ type=dict, is_optional=True, is_list=True, - nested_model="Attachments", + nested_model="Attachments2", ), "meta": FieldSchema( name="meta", type=dict, is_optional=True, is_list=False, nested_model="Meta" @@ -2433,7 +2466,7 @@ type=dict, is_optional=True, is_list=False, - nested_model="Attachments4", + nested_model="Attachments6", ), "department": FieldSchema( name="department", @@ -2751,7 +2784,7 @@ ), "description": FieldSchema(name="description", type=str, is_optional=True, is_list=False), "estimated_advertisement_date": FieldSchema( - name="estimated_advertisement_date", type=date, is_optional=True, is_list=False + name="estimated_advertisement_date", type=datetime, is_optional=True, is_list=False ), "estimated_advertisement_raw": FieldSchema( name="estimated_advertisement_raw", type=str, is_optional=True, is_list=False @@ -2805,17 +2838,17 @@ name="attachments", type=dict, is_optional=True, - is_list=False, - nested_model="Attachments3", + is_list=True, + nested_model="Attachments4", ), "bid_opening_date": FieldSchema( - name="bid_opening_date", type=date, is_optional=True, is_list=False + name="bid_opening_date", type=datetime, is_optional=True, is_list=False ), "bid_opening_raw": FieldSchema( name="bid_opening_raw", type=str, is_optional=True, is_list=False ), "category_codes": FieldSchema( - name="category_codes", type=str, is_optional=True, is_list=False + name="category_codes", type=dict, is_optional=True, is_list=True ), "contact": FieldSchema( name="contact", @@ -2824,6 +2857,9 @@ is_list=False, nested_model="GrantorContact", ), + "delisted_at": FieldSchema( + name="delisted_at", type=datetime, is_optional=True, is_list=False + ), "description": FieldSchema(name="description", type=str, is_optional=True, is_list=False), "first_seen_at": FieldSchema( name="first_seen_at", type=datetime, is_optional=True, is_list=False @@ -2852,16 +2888,18 @@ is_list=False, nested_model="Organization3", ), - "posted_date": FieldSchema(name="posted_date", type=date, is_optional=True, is_list=False), + "posted_date": FieldSchema( + name="posted_date", type=datetime, is_optional=True, is_list=False + ), "raw": FieldSchema(name="raw", type=dict, is_optional=True, is_list=False), "response_deadline": FieldSchema( - name="response_deadline", type=str, is_optional=True, is_list=False + name="response_deadline", type=datetime, is_optional=True, is_list=False ), "response_deadline_original": FieldSchema( - name="response_deadline_original", type=str, is_optional=True, is_list=False + name="response_deadline_original", type=datetime, is_optional=True, is_list=False ), "revisions": FieldSchema( - name="revisions", type=dict, is_optional=True, is_list=False, nested_model="Revisions" + name="revisions", type=dict, is_optional=True, is_list=True, nested_model="Revisions" ), "snippet": FieldSchema(name="snippet", type=str, is_optional=True, is_list=False), "solicitation_number": FieldSchema( diff --git a/tests/test_opportunity_attachments.py b/tests/test_opportunity_attachments.py new file mode 100644 index 0000000..399488a --- /dev/null +++ b/tests/test_opportunity_attachments.py @@ -0,0 +1,72 @@ +"""The `attachments(...)` expand on opportunities and notices, including the Pro-plan document-role fields.""" + +from unittest.mock import Mock, patch + +import pytest + +from tango import TangoClient +from tango.models import Notice, Opportunity +from tango.shapes.parser import ShapeParser + +ROLE_SHAPE = "title,attachments(name,doc_role,doc_role_alt)" + + +def _mock(mock_request, payload): + response = Mock() + response.is_success = True + response.json.return_value = payload + response.content = b"{}" + mock_request.return_value = response + + +def _attachments(): + return [ + {"name": "sow.pdf", "doc_role": "requirement", "doc_role_alt": None}, + {"name": "pricing.xlsx", "doc_role": "pricing", "doc_role_alt": "terms"}, + {"name": "unclassified.docx"}, + ] + + +class TestAttachmentShapes: + @pytest.mark.parametrize("model", [Opportunity, Notice]) + @pytest.mark.parametrize( + "shape", + [ + ROLE_SHAPE, + "title,attachments(name,url,extracted_text)", + "title,attachments(*)", + ], + ) + def test_attachment_shape_validates(self, model, shape): + parser = ShapeParser(cache_enabled=False) + parser.validate(parser.parse(shape), model) + + +class TestAttachmentRoles: + @pytest.mark.parametrize("method", ["get_opportunity", "get_notice"]) + @patch("tango.client.httpx.Client.request") + def test_detail_returns_document_roles(self, mock_request, method): + _mock(mock_request, {"title": "Base ops support", "attachments": _attachments()}) + row = getattr(TangoClient(api_key="k"), method)("abc", shape=ROLE_SHAPE) + assert [a.get("doc_role") for a in row["attachments"]] == ["requirement", "pricing", None] + assert row["attachments"][1]["doc_role_alt"] == "terms" + + @pytest.mark.parametrize("method", ["list_opportunities", "list_notices"]) + @patch("tango.client.httpx.Client.request") + def test_list_returns_document_roles(self, mock_request, method): + _mock( + mock_request, + { + "count": 1, + "next": None, + "previous": None, + "results": [{"title": "Amendment 1", "attachments": _attachments()}], + }, + ) + page = getattr(TangoClient(api_key="k"), method)(shape=ROLE_SHAPE) + assert mock_request.call_args.kwargs["params"]["shape"] == ROLE_SHAPE + assert [a.get("doc_role") for a in page.results[0]["attachments"]] == [ + "requirement", + "pricing", + None, + ] diff --git a/tests/test_sled.py b/tests/test_sled.py index 349c31a..46d01c5 100644 --- a/tests/test_sled.py +++ b/tests/test_sled.py @@ -6,6 +6,7 @@ the production smoke tests. """ +from datetime import UTC, datetime from unittest.mock import Mock, patch import pytest @@ -138,6 +139,55 @@ def test_results_are_parsed(self, mock_request): assert page.count == 1 assert page.results[0]["state"] == "TX" + @patch("tango.client.httpx.Client.request") + def test_delisted_at_is_requested_by_default_and_served(self, mock_request): + _mock( + mock_request, + { + "count": 1, + "next": None, + "previous": None, + "results": [ + { + "opportunity_id": "u1", + "status": "closed", + "status_reason": "delisted", + "delisted_at": "2026-09-10T06:00:00Z", + } + ], + }, + ) + page = TangoClient(api_key="k").list_sled_opportunities() + assert "delisted_at" in _call_params(mock_request)["shape"].split(",") + assert page.results[0]["delisted_at"] == datetime(2026, 9, 10, 6, 0, tzinfo=UTC) + + @patch("tango.client.httpx.Client.request") + def test_detail_fields_keep_the_api_types(self, mock_request): + _mock( + mock_request, + { + "opportunity_id": "u1", + "posted_date": "2026-09-01T14:30:00Z", + "response_deadline": "2026-10-01T17:00:00Z", + "category_codes": [{"scheme": "nigp", "code": "91000"}], + "meta": {"jurisdiction_declared": False, "last_change_source_declared": True}, + "attachments": [{"name": "rfp.pdf", "size_bytes": 1024, "pages": 3}], + }, + ) + row = TangoClient(api_key="k").get_sled_opportunity( + "u1", + shape=( + "opportunity_id,posted_date,response_deadline,category_codes," + "meta(jurisdiction_declared,last_change_source_declared)," + "attachments(name,size_bytes,pages)" + ), + ) + assert row["posted_date"] == datetime(2026, 9, 1, 14, 30, tzinfo=UTC) + assert row["response_deadline"] == datetime(2026, 10, 1, 17, 0, tzinfo=UTC) + assert row["category_codes"] == [{"scheme": "nigp", "code": "91000"}] + assert row["meta"]["jurisdiction_declared"] is False + assert row["attachments"][0]["size_bytes"] == 1024 + class TestSledRevisions: @patch("tango.client.httpx.Client.request") @@ -263,6 +313,17 @@ def test_the_document_body_validates_when_named(self): SledOpportunity, ) + @pytest.mark.parametrize( + "shape", + [ + "opportunity_id,status,status_reason,delisted_at", + "opportunity_id,meta(jurisdiction_declared,last_change_source_declared)", + ], + ) + def test_fields_added_after_launch_validate(self, shape): + parser = ShapeParser(cache_enabled=True) + parser.validate(parser.parse(shape), SledOpportunity) + def test_no_default_shape_names_the_paid_document_body(self): """The API resolves the body only when named, so a default shape that named it would make every detail fetch pay for it.""" for shape in (