From 800216ba4e9226e5782ba8e4c658dea447c4be6b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 13:00:10 +0000 Subject: [PATCH 1/4] Eliminate deprecated utcnow() and fix a local-time deadline bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enabling ruff's DTZ rules found a real correctness bug, not just style. `openfoia request send` wrote `req.sent_at = datetime.now()` — LOCAL time into a column every reader treats as UTC. `days_pending()` and `is_overdue()` compare it against UTC, so for a user in UTC+9 or UTC-8 the statutory FOIA deadline was off by up to a day. For a tool whose whole job is tracking those deadlines, that matters. - Add `models.utcnow()`: UTC as a **naive** datetime. The ORM's DateTime columns are timezone-naive, so returning an aware value would raise "can't subtract offset-naive and offset-aware datetimes" against every existing row. This removes the deprecated `datetime.utcnow()` (CI already targets 3.13) while preserving the storage convention exactly. Moving to timezone-aware columns is a separate, deliberate migration. - Migrate all 21 `utcnow()` call sites across 9 modules, including an aliased `dt.utcnow()` that a naive search missed. - Fix the `sent_at` local-time bug; make request-number dates UTC so they match the agent-generated ones. - Where local time IS correct — the date printed on an outgoing letter, a file listing shown to the user — keep it and annotate why, rather than converting and silently changing what the user sees. Tests pin the convention: the helper is naive, is UTC, emits no deprecation warning, `days_pending`/`is_overdue` still work against naive storage, no attribute-style `.utcnow()` calls remain, and no local time is assigned to a persisted timestamp column. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KWzqFx1d2uB6cjQV81mgJh --- openfoia/agent.py | 9 +- openfoia/campaign.py | 9 +- openfoia/cli.py | 155 +++++++++++------- openfoia/db.py | 2 +- openfoia/ftm.py | 3 +- openfoia/ftm_import.py | 4 +- openfoia/gateways/__init__.py | 2 +- openfoia/gateways/email.py | 16 +- openfoia/gateways/fax.py | 4 +- .../migrations/versions/001_initial_schema.py | 3 +- openfoia/models.py | 37 ++++- openfoia/pipeline/__init__.py | 2 +- openfoia/pipeline/extract.py | 5 +- openfoia/pipeline/ingest.py | 5 +- openfoia/pipeline/ocr.py | 2 +- openfoia/pipeline/web.py | 1 - openfoia/records/__init__.py | 12 +- openfoia/records/sec_edgar.py | 1 - openfoia/security.py | 3 +- openfoia/server.py | 32 ++-- openfoia/templates.py | 12 +- openfoia/tor_browse.py | 5 +- tests/test_datetime_semantics.py | 112 +++++++++++++ 23 files changed, 314 insertions(+), 122 deletions(-) create mode 100644 tests/test_datetime_semantics.py diff --git a/openfoia/agent.py b/openfoia/agent.py index 1657cd4..b7e81fc 100644 --- a/openfoia/agent.py +++ b/openfoia/agent.py @@ -18,6 +18,7 @@ from typing import Any from .models import Agency, Request, RequestStatus +from .models import utcnow as _utcnow logger = logging.getLogger(__name__) @@ -385,6 +386,7 @@ async def _draft_request(self, params: dict[str, Any]) -> dict[str, Any]: """ import uuid + from .models import DeliveryMethod, User request_id = str(uuid.uuid4()) @@ -401,7 +403,7 @@ async def _draft_request(self, params: dict[str, Any]) -> dict[str, Any]: if user and agency: from datetime import datetime as dt - req_num = f"REQ-{dt.utcnow().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}" + req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}" new_req = Request( id=request_id, request_number=req_num, @@ -432,7 +434,7 @@ async def _send_request(self, params: dict[str, Any]) -> dict[str, Any]: return {"error": f"Request not found: {request_id}"} request.status = RequestStatus.SENT - request.sent_at = datetime.utcnow() + request.sent_at = _utcnow() # Auto-set due date (20 business days per FOIA statute) if not request.due_date: @@ -541,6 +543,7 @@ async def _process_document(self, params: dict[str, Any]) -> dict[str, Any]: database, where it becomes visible in reports and exports. """ from pathlib import Path + from .db import get_data_dir doc_path = params.get("document_path", "") @@ -604,7 +607,7 @@ async def _extract_entities(self, params: dict[str, Any]) -> dict[str, Any]: async def _build_entity_graph(self, params: dict[str, Any]) -> dict[str, Any]: """Build entity graph.""" - from .models import Entity, Document, entity_links + from .models import Document, Entity, entity_links query = self.db.query(Entity) request_ids = params.get("request_ids") diff --git a/openfoia/campaign.py b/openfoia/campaign.py index 5c4c669..da7777c 100644 --- a/openfoia/campaign.py +++ b/openfoia/campaign.py @@ -23,6 +23,7 @@ RequestStatus, User, ) +from .models import utcnow as _utcnow def _sandbox_env() -> SandboxedEnvironment: @@ -71,7 +72,7 @@ def render( context = { "participant": participant, "agency": agency, - "date": datetime.utcnow().strftime("%B %d, %Y"), + "date": _utcnow().strftime("%B %d, %Y"), "custom": custom_params or {}, } @@ -167,7 +168,7 @@ async def generate_request( ) # Generate request number - request_number = f"REQ-{datetime.utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" + request_number = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" # Determine delivery method if agency.foia_email and template.recommended_method == DeliveryMethod.EMAIL: @@ -211,7 +212,7 @@ async def schedule_staggered_send( 2. Overwhelming agency systems 3. Making it easy to identify and block """ - start = start_time or datetime.utcnow() + start = start_time or _utcnow() schedule = [] # Distribute evenly with some randomness @@ -307,7 +308,7 @@ async def generate_progress_report(self, campaign: Campaign) -> str: - **Active:** {"Yes" if stats["is_active"] else "No"} --- -*Generated: {datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")}* +*Generated: {_utcnow().strftime("%Y-%m-%d %H:%M UTC")}* """.strip() diff --git a/openfoia/cli.py b/openfoia/cli.py index 6a62f9f..2b13069 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -14,6 +14,8 @@ from rich.progress import Progress, SpinnerColumn, TextColumn from rich.table import Table +from .models import utcnow as _utcnow + app = typer.Typer( name="openfoia", help="Crowdsourced FOIA automation with AI-powered document analysis.", @@ -71,7 +73,7 @@ def init( openfoia init --encrypt # Initialize with encryption (prompts) openfoia init --encrypt --duress # Also set up a decoy database """ - from .db import get_data_dir, get_db_path, init_db, has_sqlcipher + from .db import get_data_dir, get_db_path, has_sqlcipher, init_db data_dir = get_data_dir() @@ -113,7 +115,7 @@ def init( # Show stats from .db import get_session - from .models import Agency, Request, Document + from .models import Agency, Document, Request with get_session(password=password) as session: agency_count = session.query(Agency).count() @@ -405,7 +407,7 @@ def serve( import secrets import socket - from .browser import detect_browsers, launch_browser, print_browser_menu, BrowserType + from .browser import BrowserType, detect_browsers, launch_browser, print_browser_menu # Generate session token for security token = secrets.token_urlsafe(16) @@ -473,8 +475,8 @@ def serve( rprint("[yellow]No browser auto-selected. Copy the URL above.[/yellow]\n") # Start the server - from .server import run_server from .db import get_data_dir + from .server import run_server run_server(host=host, port=port, token=token, data_dir=get_data_dir()) @@ -556,7 +558,7 @@ def encrypt( openfoia db encrypt --password SECRET openfoia db encrypt # will prompt for password """ - from .db import get_db_path, encrypt_database, has_sqlcipher + from .db import encrypt_database, get_db_path, has_sqlcipher if not has_sqlcipher(): rprint("[bold red]Error:[/bold red] pysqlcipher3 is not installed.") @@ -688,13 +690,18 @@ def request_new( ): """Create a new FOIA request.""" from uuid import uuid4 + from .db import get_db_path, get_session, init_db from .models import ( Agency as AgencyModel, - Request as RequestModel, - User, - RequestStatus, + ) + from .models import ( DeliveryMethod, + RequestStatus, + User, + ) + from .models import ( + Request as RequestModel, ) if body_file: @@ -735,7 +742,7 @@ def request_new( session.flush() # Create request - req_num = f"REQ-{datetime.now().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" + req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" try: delivery = DeliveryMethod(method.lower()) @@ -781,8 +788,10 @@ def request_list( limit: int = typer.Option(20, "--limit", "-n", help="Maximum results"), ): """List FOIA requests.""" - from .db import get_session, get_db_path - from .models import Request as RequestModel, Agency as AgencyModel, RequestStatus + from .db import get_db_path, get_session + from .models import Agency as AgencyModel + from .models import Request as RequestModel + from .models import RequestStatus db_path = get_db_path() if not db_path.exists(): @@ -849,8 +858,9 @@ def request_status( request_id: str = typer.Argument(..., help="Request ID or number"), ): """Check status of a FOIA request.""" - from .db import get_session, get_db_path - from .models import Request as RequestModel, TimelineEvent + from .db import get_db_path, get_session + from .models import Request as RequestModel + from .models import TimelineEvent db_path = get_db_path() if not db_path.exists(): @@ -971,9 +981,10 @@ def request_send( openfoia request send -a FBI -s "Test" -t standard -n "Test User" -e test@example.com --dry-run """ import asyncio + from .db import get_db_path, get_session - from .models import Agency from .gateways.base import DeliveryPayload + from .models import Agency if method not in ("email", "fax", "mail"): rprint(f"[red]Unknown method '{method}'. Use: email, fax, mail[/red]") @@ -1022,7 +1033,7 @@ def request_send( # Get body content if template: - from .templates import standard_request, records_about_self, RequesterInfo, RequestDetails + from .templates import RequestDetails, RequesterInfo, records_about_self, standard_request requester = RequesterInfo(name=name, email=email) details = RequestDetails(subject=subject, description=subject) @@ -1089,8 +1100,8 @@ def request_send( from .db import get_data_dir config_path = get_data_dir() / "config.json" - import os import json + import os config = {} if config_path.exists(): @@ -1206,10 +1217,13 @@ def request_send( rprint(f" Expected delivery: {result.metadata['expected_delivery_date']}") # Update the matching Request in the DB if one exists - from .db import get_session, get_db_path - from .models import Request as RequestModel, RequestStatus, Agency as AgencyModel from datetime import timedelta + from .db import get_db_path, get_session + from .models import Agency as AgencyModel + from .models import Request as RequestModel + from .models import RequestStatus + db_path = get_db_path() if db_path.exists(): with get_session() as session: @@ -1227,7 +1241,7 @@ def request_send( ) if req: req.status = RequestStatus.SENT - req.sent_at = datetime.now() + req.sent_at = _utcnow() req.delivery_reference = result.reference_id # Auto-set due date response_days = req.agency.typical_response_days if req.agency else 20 @@ -1277,6 +1291,7 @@ def docs_ingest( openfoia docs ingest ./doc.pdf --keep-metadata """ import asyncio + from .db import get_data_dir, get_db_path, init_db from .pipeline.ingest import DocumentIngester @@ -1359,9 +1374,10 @@ def docs_ingest( # Persist Document rows to database if results: + from uuid import uuid4 + from .db import get_session from .models import Document, DocumentType - from uuid import uuid4 with get_session() as session: for r in results: @@ -1380,7 +1396,7 @@ def docs_ingest( # Check if request_id is valid, otherwise create without it if not request_id: # Create a placeholder request for unassociated documents - from .models import Request, User, RequestStatus, DeliveryMethod + from .models import DeliveryMethod, Request, RequestStatus, User user = session.query(User).first() if not user: @@ -1464,6 +1480,7 @@ def docs_ocr( openfoia docs ocr document.pdf --backend google """ import asyncio + from .pipeline.ocr import OCREngine, RedactionDetector if not file_path.exists(): @@ -1544,7 +1561,7 @@ def agency_list( limit: int = typer.Option(50, "--limit", "-n", help="Maximum results"), ): """List agencies in the database.""" - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Agency, AgencyLevel db_path = get_db_path() @@ -1598,7 +1615,7 @@ def agency_search( limit: int = typer.Option(20, "--limit", "-n", help="Maximum results"), ): """Search for agencies by name or abbreviation.""" - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Agency db_path = get_db_path() @@ -1642,7 +1659,7 @@ def agency_info( agency_id: str = typer.Argument(..., help="Agency abbreviation or name"), ): """Show detailed information about an agency.""" - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Agency db_path = get_db_path() @@ -1745,7 +1762,7 @@ def template_generate( openfoia template generate standard -a FBI -s "Records on X" -n "Jane Doe" -e jane@example.com openfoia template generate standard -a EPA -s "Pollution data" -n "John Smith" -e john@example.com -j """ - from .templates import standard_request, records_about_self, RequesterInfo, RequestDetails + from .templates import RequestDetails, RequesterInfo, records_about_self, standard_request # Build requester info requester = RequesterInfo( @@ -1889,6 +1906,7 @@ def campaign_create( ): """Create a new crowdsourced campaign.""" from uuid import uuid4 + from .db import get_db_path, get_session, init_db from .models import Campaign, User @@ -1932,7 +1950,7 @@ def campaign_create( @campaign_app.command("list") def campaign_list(): """List all campaigns.""" - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Campaign db_path = get_db_path() @@ -1974,7 +1992,7 @@ def campaign_status( campaign_id: str = typer.Argument(..., help="Campaign ID (or prefix)"), ): """Check campaign progress.""" - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Campaign, RequestStatus db_path = get_db_path() @@ -2035,7 +2053,8 @@ def campaign_join( ): """Join a campaign as a participant.""" from uuid import uuid4 - from .db import get_session, get_db_path + + from .db import get_db_path, get_session from .models import Campaign, User db_path = get_db_path() @@ -2091,13 +2110,18 @@ def campaign_distribute( target agency and assigns them round-robin to participants. """ from uuid import uuid4 - from .db import get_session, get_db_path + + from .db import get_db_path, get_session from .models import ( - Campaign, Agency as AgencyModel, - Request as RequestModel, - RequestStatus, + ) + from .models import ( + Campaign, DeliveryMethod, + RequestStatus, + ) + from .models import ( + Request as RequestModel, ) db_path = get_db_path() @@ -2163,7 +2187,7 @@ def campaign_distribute( skipped += 1 continue - req_num = f"REQ-{datetime.now().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" + req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" request = RequestModel( id=str(uuid4()), request_number=req_num, @@ -2195,8 +2219,9 @@ def campaign_progress( campaign_id: str = typer.Argument(..., help="Campaign ID (or prefix)"), ): """Show per-participant, per-agency status grid for a campaign.""" - from .db import get_session, get_db_path - from .models import Campaign, Agency as AgencyModel + from .db import get_db_path, get_session + from .models import Agency as AgencyModel + from .models import Campaign db_path = get_db_path() if not db_path.exists(): @@ -2305,7 +2330,7 @@ def analyze_extract( Pipeline: regex + NER → merge → LLM validation (if available). Use --ensemble to run ALL NER backends (GLiNER + spaCy) together. """ - from .db import get_session, get_db_path + from .db import get_db_path, get_session from .models import Document, Entity db_path = get_db_path() @@ -2355,6 +2380,7 @@ def analyze_extract( # Run extraction import asyncio + from .pipeline.extract import EntityExtractor extractor = EntityExtractor(model=model) if model else EntityExtractor() @@ -2385,6 +2411,7 @@ def analyze_extract( # Save entities to database from uuid import uuid4 + from .models import entity_links entity_id_map: dict[str, str] = {} # normalized_text.lower() -> entity.id @@ -2521,7 +2548,10 @@ def analyze_graphs_list(): size = f.stat().st_size size_str = f"{size / 1024:.0f}KB" if size > 1024 else f"{size}B" - modified = datetime.fromtimestamp(f.stat().st_mtime).strftime("%Y-%m-%d %H:%M") + # Local time is intended: this is a file listing shown to the user. + modified = datetime.fromtimestamp( # noqa: DTZ006 + f.stat().st_mtime + ).strftime("%Y-%m-%d %H:%M") table.add_row(name, " + ".join(types), size_str, modified) @@ -2559,8 +2589,9 @@ def analyze_graph( openfoia analyze graph --request REQ-001 --name epa # filter + save openfoia analyze graphs # list saved graphs """ - from .db import get_session, get_db_path - from .models import Entity, Document, Request as RequestModel, entity_links + from .db import get_db_path, get_session + from .models import Document, Entity, entity_links + from .models import Request as RequestModel db_path = get_db_path() if not db_path.exists(): @@ -2627,9 +2658,11 @@ def analyze_graph( doc_ids = {e.document_id for e in entities if e.document_id} documents = {} if doc_ids: - from .models import Document as DocModel, Request as ReqModel import re as re_mod + from .models import Document as DocModel + from .models import Request as ReqModel + for doc in session.query(DocModel).filter(DocModel.id.in_(doc_ids)).all(): # Derive source URL from request body or filename source_url = None @@ -2858,8 +2891,8 @@ def _fuzzy_match_column(header: str) -> str | None: def _llm_map_columns(headers: list[str], sample_rows: list[list[str]]) -> dict[str, int] | None: """Use the configured LLM to figure out which columns map to name/pattern/description.""" - from .pipeline.extract import _llm_available, _call_ollama from .config import load_config + from .pipeline.extract import _call_ollama, _llm_available cfg = load_config() if not _llm_available(cfg.ai.provider, cfg.ai.api_key, cfg.ai.base_url): @@ -2913,8 +2946,8 @@ def _llm_generate_regex(description: str) -> str | None: 2. Matches at least one example from the description (if examples are present) 3. Is reasonably short (not hallucinated garbage) """ - from .pipeline.extract import _llm_available from .config import load_config + from .pipeline.extract import _llm_available cfg = load_config() if not _llm_available(cfg.ai.provider, cfg.ai.api_key, cfg.ai.base_url): @@ -3303,8 +3336,10 @@ def deadline_list( Federal agencies have 20 business days to respond (5 U.S.C. 552). This command shows what's due, what's overdue, and what needs follow-up. """ - from .db import get_session, get_db_path - from .models import Request as RequestModel, Agency as AgencyModel, RequestStatus + from .db import get_db_path, get_session + from .models import Agency as AgencyModel + from .models import Request as RequestModel + from .models import RequestStatus db_path = get_db_path() if not db_path.exists(): @@ -3362,7 +3397,7 @@ def deadline_list( table.add_column("Days Over", style="red") for r in overdue: - days_over = (datetime.utcnow() - r.due_date).days + days_over = (_utcnow() - r.due_date).days table.add_row( r.request_number, r.agency.abbreviation or r.agency.name, @@ -3384,7 +3419,7 @@ def deadline_list( table.add_column("Days Left", style="green") for r in upcoming: - days_left = (r.due_date - datetime.utcnow()).days + days_left = (r.due_date - _utcnow()).days color = "green" if days_left > 5 else "yellow" table.add_row( r.request_number, @@ -3411,8 +3446,9 @@ def deadline_check(): Example (add to .bashrc): openfoia deadlines check 2>/dev/null """ - from .db import get_session, get_db_path - from .models import Request as RequestModel, RequestStatus + from .db import get_db_path, get_session + from .models import Request as RequestModel + from .models import RequestStatus db_path = get_db_path() if not db_path.exists(): @@ -3440,7 +3476,7 @@ def deadline_check(): r.due_date = _foia_due_date(r.sent_at) if r.is_overdue(): overdue_count += 1 - days_over = (datetime.utcnow() - r.due_date).days + days_over = (_utcnow() - r.due_date).days rprint(f"[red]OVERDUE:[/red] {r.request_number} — {r.subject} (+{days_over} days)") if overdue_count: @@ -3481,6 +3517,7 @@ def browse( openfoia browse https://example.com --tor --headless --save # Headless Tor """ import asyncio + from .tor_browse import browse as _browse try: @@ -3542,6 +3579,7 @@ def purge( print_ssd_warning, secure_delete_dir, ) + from .db import get_data_dir as _get_data_dir data_dir = _get_data_dir() @@ -3636,6 +3674,7 @@ def ingest_url( openfoia ingest --url https://example.onion/docs --tor """ import asyncio + from .db import get_data_dir from .pipeline.web import archive_url @@ -3722,6 +3761,7 @@ def records_search( openfoia records search "EPA water" --source muckrock """ import asyncio + from .records import get_adapter, list_sources # Validate source @@ -4119,10 +4159,11 @@ def records_download( # Persist Document rows to database if ingest_results: - from .db import get_session - from .models import Document, DocumentType, Request, User, RequestStatus, DeliveryMethod from uuid import uuid4 + from .db import get_session + from .models import DeliveryMethod, Document, DocumentType, Request, RequestStatus, User + with get_session() as session: # Create a placeholder request for downloaded docs user = session.query(User).first() @@ -4209,9 +4250,10 @@ def crossref( openfoia crossref --icij-data ./icij-csvs/ # include Offshore Leaks openfoia crossref --ftm results.ftm.json # export as FollowTheMoney """ - from .db import get_session, get_db_path - from .models import Entity, Document, Request as RequestModel from .crossref import crossref_entities + from .db import get_db_path, get_session + from .models import Document, Entity + from .models import Request as RequestModel db_path = get_db_path() if not db_path.exists(): @@ -4406,10 +4448,11 @@ def analyze_export( openfoia analyze export openfoia analyze export -o investigation.ftm.json -r REQ-20260322-ABC """ - from .db import get_session, get_db_path - from .models import Entity, Document, Request as RequestModel, entity_links - from .pipeline.extract import ExtractedEntity + from .db import get_db_path, get_session from .ftm import export_ftm + from .models import Document, Entity, entity_links + from .models import Request as RequestModel + from .pipeline.extract import ExtractedEntity db_path = get_db_path() if not db_path.exists(): diff --git a/openfoia/db.py b/openfoia/db.py index 60b39c1..235d9d5 100644 --- a/openfoia/db.py +++ b/openfoia/db.py @@ -195,7 +195,7 @@ def get_db_path(password: str | None = None) -> Path: password = get_db_password() if password: - from .security import is_duress_password, get_decoy_db_path + from .security import get_decoy_db_path, is_duress_password if is_duress_password(password): return get_decoy_db_path() diff --git a/openfoia/ftm.py b/openfoia/ftm.py index 18c1ed1..622f1a4 100644 --- a/openfoia/ftm.py +++ b/openfoia/ftm.py @@ -10,12 +10,11 @@ from __future__ import annotations import json -from typing import Any from pathlib import Path +from typing import Any from .models import EntityType - # Map OpenFOIA entity types to FtM schema types _ENTITY_TYPE_TO_FTM_SCHEMA: dict[str, str] = { "person": "Person", diff --git a/openfoia/ftm_import.py b/openfoia/ftm_import.py index fd36b87..0181c66 100644 --- a/openfoia/ftm_import.py +++ b/openfoia/ftm_import.py @@ -192,7 +192,7 @@ def import_ftm_to_db( Returns: (entities_imported, relationships_imported) """ - from .db import get_session, get_db_path, init_db + from .db import get_db_path, get_session, init_db from .models import Entity db_path = get_db_path() @@ -293,7 +293,7 @@ def _get_or_create_import_doc(session: Any, tag: str | None = None) -> str: if _IMPORT_DOC_ID: return _IMPORT_DOC_ID - from .models import Document, DocumentType, Request, User, Agency, RequestStatus, DeliveryMethod + from .models import Agency, DeliveryMethod, Document, DocumentType, Request, RequestStatus, User # Need a request to hang the document on user = session.query(User).first() diff --git a/openfoia/gateways/__init__.py b/openfoia/gateways/__init__.py index a1d45f8..bdf570c 100644 --- a/openfoia/gateways/__init__.py +++ b/openfoia/gateways/__init__.py @@ -8,9 +8,9 @@ """ from .base import DeliveryGateway, DeliveryResult +from .email import EmailGateway from .fax import TwilioFaxGateway from .mail import LobMailGateway -from .email import EmailGateway __all__ = [ "DeliveryGateway", diff --git a/openfoia/gateways/email.py b/openfoia/gateways/email.py index b1b8a36..cce6749 100644 --- a/openfoia/gateways/email.py +++ b/openfoia/gateways/email.py @@ -11,6 +11,7 @@ from email.mime.text import MIMEText from email.utils import getaddresses +from ..models import utcnow as _utcnow from .base import DeliveryGateway, DeliveryPayload, DeliveryResult, DeliveryStatus @@ -138,13 +139,13 @@ def _send(): import hashlib ref_id = hashlib.sha256( - f"{payload.recipient_address}:{payload.subject}:{datetime.utcnow().isoformat()}".encode() + f"{payload.recipient_address}:{payload.subject}:{_utcnow().isoformat()}".encode() ).hexdigest()[:16] return DeliveryResult( status=DeliveryStatus.SENT, reference_id=ref_id, - sent_at=datetime.utcnow(), + sent_at=_utcnow(), cost_cents=0, # Email is free (sort of) metadata={ "to": payload.recipient_address, @@ -164,16 +165,17 @@ def _send(): async def _send_sendgrid(self, payload: DeliveryPayload) -> DeliveryResult: """Send via SendGrid API.""" try: + import base64 + from sendgrid import SendGridAPIClient from sendgrid.helpers.mail import ( - Mail, Attachment, + Disposition, FileContent, FileName, FileType, - Disposition, + Mail, ) - import base64 message = Mail( from_email=(self.from_email, self.from_name), @@ -206,7 +208,7 @@ async def _send_sendgrid(self, payload: DeliveryPayload) -> DeliveryResult: return DeliveryResult( status=DeliveryStatus.SENT, reference_id=message_id, - sent_at=datetime.utcnow(), + sent_at=_utcnow(), cost_cents=0, metadata={ "to": payload.recipient_address, @@ -256,7 +258,7 @@ def _format_email_body(self, payload: DeliveryPayload) -> str: --- REQUEST DETAILS Subject: {payload.subject} -Date: {datetime.utcnow().strftime("%B %d, %Y")} +Date: {_utcnow().strftime("%B %d, %Y")} I request a fee waiver for this request. Disclosure of the requested information is in the public interest because it is likely to contribute significantly to public understanding of government operations and activities. diff --git a/openfoia/gateways/fax.py b/openfoia/gateways/fax.py index 82267bb..41e459e 100644 --- a/openfoia/gateways/fax.py +++ b/openfoia/gateways/fax.py @@ -268,9 +268,9 @@ def _generate_fax_pdf(self, payload: DeliveryPayload) -> bytes: def _generate_pdf_reportlab(self, payload: DeliveryPayload) -> bytes: """Generate PDF using reportlab with proper legal formatting.""" from reportlab.lib.pagesizes import letter - from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle + from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet from reportlab.lib.units import inch - from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer + from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer buffer = io.BytesIO() doc = SimpleDocTemplate( diff --git a/openfoia/migrations/versions/001_initial_schema.py b/openfoia/migrations/versions/001_initial_schema.py index 2f40e6b..f61f6e3 100644 --- a/openfoia/migrations/versions/001_initial_schema.py +++ b/openfoia/migrations/versions/001_initial_schema.py @@ -8,9 +8,8 @@ from typing import Sequence, Union -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision: str = "001" diff --git a/openfoia/models.py b/openfoia/models.py index 9a9d2d9..26a9c3e 100644 --- a/openfoia/models.py +++ b/openfoia/models.py @@ -11,7 +11,26 @@ from __future__ import annotations import enum -from datetime import datetime +from datetime import datetime, timezone + + +def utcnow() -> datetime: + """Current UTC time as a **naive** datetime. + + The ORM's ``DateTime`` columns are timezone-naive, and model helpers such + as ``Request.days_pending`` compare stored values against "now". Returning + an aware datetime here would raise + ``TypeError: can't subtract offset-naive and offset-aware datetimes`` + against every existing row. + + This replaces the deprecated ``utcnow()`` (removed in a future + Python; CI already targets 3.13) while preserving the naive-UTC storage + convention exactly. Moving to timezone-aware columns is a separate, + deliberate migration. + """ + return datetime.now(timezone.utc).replace(tzinfo=None) + + from typing import Any from uuid import uuid4 @@ -24,8 +43,8 @@ ForeignKey, Integer, String, - Text, Table, + Text, create_engine, ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship @@ -135,7 +154,7 @@ class User(Base): name: Mapped[str] = mapped_column(String(255)) organization: Mapped[str | None] = mapped_column(String(255), nullable=True) is_journalist: Mapped[bool] = mapped_column(default=False) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) # Relationships requests: Mapped[list["Request"]] = relationship(back_populates="requester") @@ -209,7 +228,7 @@ class Request(Base): agency_tracking_number: Mapped[str | None] = mapped_column(String(100), nullable=True) # Dates - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) due_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) @@ -234,13 +253,13 @@ def days_pending(self) -> int: """Days since request was sent.""" if not self.sent_at: return 0 - return (datetime.utcnow() - self.sent_at).days + return (utcnow() - self.sent_at).days def is_overdue(self) -> bool: """Whether the request is past its due date.""" if not self.due_date: return False - return datetime.utcnow() > self.due_date + return utcnow() > self.due_date class Document(Base): @@ -271,7 +290,7 @@ class Document(Base): ) # e.g., ["b(6)", "b(7)(A)"] # Dates - received_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + received_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) processed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) # Relationships @@ -333,7 +352,7 @@ class Campaign(Base): # Status is_active: Mapped[bool] = mapped_column(default=True) - created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) # Relationships @@ -364,7 +383,7 @@ class TimelineEvent(Base): String(50) ) # sent, acknowledged, response, appeal, etc. description: Mapped[str] = mapped_column(Text) - occurred_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + occurred_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) # Relationships diff --git a/openfoia/pipeline/__init__.py b/openfoia/pipeline/__init__.py index d2196c9..07a1163 100644 --- a/openfoia/pipeline/__init__.py +++ b/openfoia/pipeline/__init__.py @@ -8,9 +8,9 @@ 5. Link - Connect entities across documents """ +from .extract import EntityExtractor from .ingest import DocumentIngester from .ocr import OCREngine -from .extract import EntityExtractor __all__ = [ "DocumentIngester", diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py index d260c05..f4e723f 100644 --- a/openfoia/pipeline/extract.py +++ b/openfoia/pipeline/extract.py @@ -16,9 +16,8 @@ from dataclasses import dataclass, field from typing import Any -from ..config import load_config, OpenFOIAConfig, EntityConfig -from ..models import EntityType, ConfidenceLevel - +from ..config import EntityConfig, OpenFOIAConfig, load_config +from ..models import ConfidenceLevel, EntityType # --------------------------------------------------------------------------- # Data classes diff --git a/openfoia/pipeline/ingest.py b/openfoia/pipeline/ingest.py index 31e847d..beac585 100644 --- a/openfoia/pipeline/ingest.py +++ b/openfoia/pipeline/ingest.py @@ -13,6 +13,7 @@ from uuid import uuid4 from ..models import DocumentType +from ..models import utcnow as _utcnow @dataclass @@ -103,7 +104,7 @@ async def ingest_file( checksum=checksum, metadata={ "original_path": str(file_path), - "ingested_at": datetime.utcnow().isoformat(), + "ingested_at": _utcnow().isoformat(), "doc_type": doc_type.value, "request_id": request_id, "metadata_stripped": stripped_info if stripped_info else None, @@ -159,7 +160,7 @@ async def ingest_bytes( extracted_text=extracted_text, checksum=checksum, metadata={ - "ingested_at": datetime.utcnow().isoformat(), + "ingested_at": _utcnow().isoformat(), "doc_type": doc_type.value, "request_id": request_id, **(metadata or {}), diff --git a/openfoia/pipeline/ocr.py b/openfoia/pipeline/ocr.py index 91d7e83..e232358 100644 --- a/openfoia/pipeline/ocr.py +++ b/openfoia/pipeline/ocr.py @@ -358,8 +358,8 @@ async def _count_visual_redactions(self, pdf_path: Path) -> int: Uses image analysis to detect large black rectangular regions. """ - from pdf2image import convert_from_path import numpy as np + from pdf2image import convert_from_path def _analyze(): with tempfile.TemporaryDirectory(dir=get_ocr_temp_dir()) as scratch: diff --git a/openfoia/pipeline/web.py b/openfoia/pipeline/web.py index 54a09cb..5e39069 100644 --- a/openfoia/pipeline/web.py +++ b/openfoia/pipeline/web.py @@ -15,7 +15,6 @@ from typing import Any from uuid import uuid4 - # Known tracker/analytics domains and script patterns to strip TRACKER_PATTERNS: list[str] = [ r"google-analytics\.com", diff --git a/openfoia/records/__init__.py b/openfoia/records/__init__.py index 0cc57c9..8ac3f02 100644 --- a/openfoia/records/__init__.py +++ b/openfoia/records/__init__.py @@ -39,15 +39,15 @@ def list_sources() -> list[str]: def _auto_register() -> None: """Auto-register built-in adapters.""" - from .opencorporates import OpenCorporatesAdapter - from .sec_edgar import SECEdgarAdapter - from .muckrock import MuckRockAdapter from .documentcloud import DocumentCloudAdapter - from .usaspending import USASpendingAdapter - from .propublica_nonprofit import ProPublicaNonprofitAdapter - from .govinfo import GovInfoAdapter from .fec import FECAdapter + from .govinfo import GovInfoAdapter + from .muckrock import MuckRockAdapter + from .opencorporates import OpenCorporatesAdapter + from .propublica_nonprofit import ProPublicaNonprofitAdapter from .regulations import RegulationsGovAdapter + from .sec_edgar import SECEdgarAdapter + from .usaspending import USASpendingAdapter register("opencorporates", OpenCorporatesAdapter) register("sec", SECEdgarAdapter) diff --git a/openfoia/records/sec_edgar.py b/openfoia/records/sec_edgar.py index 1e20bf8..c7ab652 100644 --- a/openfoia/records/sec_edgar.py +++ b/openfoia/records/sec_edgar.py @@ -8,7 +8,6 @@ from typing import Any - from .base import AdapterRequestError, RecordAdapter, RecordEntity, SearchResult EFTS_BASE = "https://efts.sec.gov/LATEST/search-index" diff --git a/openfoia/security.py b/openfoia/security.py index fc96940..cb5dd3b 100644 --- a/openfoia/security.py +++ b/openfoia/security.py @@ -20,6 +20,7 @@ from rich import print as rprint +from .models import utcnow as _utcnow # --------------------------------------------------------------------------- # Secure file deletion @@ -510,7 +511,7 @@ def _creator(): session.flush() # --- Requests (bland, non-sensitive) --- - now = datetime.utcnow() + now = _utcnow() requests_data = [ { "subject": "Monthly weather data summaries for Portland, OR (2024)", diff --git a/openfoia/server.py b/openfoia/server.py index b55eb3d..4bf8b6c 100644 --- a/openfoia/server.py +++ b/openfoia/server.py @@ -11,14 +11,15 @@ from pathlib import Path from uuid import uuid4 -from fastapi import FastAPI, Request, HTTPException, Depends, Query, UploadFile, File -from fastapi.responses import HTMLResponse +from fastapi import Depends, FastAPI, File, HTTPException, Query, Request, UploadFile from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import HTMLResponse from fastapi.staticfiles import StaticFiles -from starlette.middleware.trustedhost import TrustedHostMiddleware from pydantic import BaseModel from sqlalchemy import func +from starlette.middleware.trustedhost import TrustedHostMiddleware +from .models import utcnow as _utcnow #: Hard cap on a single uploaded document (100 MiB), matching CLI ingest. MAX_UPLOAD_BYTES = 100 * 1024 * 1024 @@ -138,11 +139,13 @@ async def stats(token: str = Depends(verify_token)): """Get overview statistics.""" from .db import get_session from .models import ( - Request as FOIARequest, Document, Entity, - RequestStatus, EntityType, + RequestStatus, + ) + from .models import ( + Request as FOIARequest, ) with get_session() as session: @@ -225,7 +228,8 @@ async def list_requests( ): """List FOIA requests.""" from .db import get_session - from .models import Request as FOIARequest, RequestStatus, Agency + from .models import Agency, RequestStatus + from .models import Request as FOIARequest with get_session() as session: query = session.query(FOIARequest).join(Agency) @@ -267,11 +271,13 @@ async def create_request( """Create a new FOIA request.""" from .db import get_session from .models import ( - Request as FOIARequest, Agency, - User, - RequestStatus, DeliveryMethod, + RequestStatus, + User, + ) + from .models import ( + Request as FOIARequest, ) with get_session() as session: @@ -290,7 +296,7 @@ async def create_request( session.add(user) session.flush() - req_num = f"REQ-{datetime.utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" + req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}" try: method = DeliveryMethod(request_data.method) @@ -514,7 +520,8 @@ async def list_entities( async def deadlines(token: str = Depends(verify_token)): """Get upcoming deadlines for pending requests.""" from .db import get_session - from .models import Request as FOIARequest, RequestStatus, Agency + from .models import Agency, RequestStatus + from .models import Request as FOIARequest with get_session() as session: active_statuses = [ @@ -560,7 +567,7 @@ async def get_graph( ): """Get entity relationship graph.""" from .db import get_session - from .models import Entity, Document, entity_links + from .models import Document, Entity, entity_links with get_session() as session: q = session.query(Entity) @@ -1244,6 +1251,7 @@ def run_server( ) -> None: """Run the OpenFOIA server.""" import socket + import uvicorn # Generate token if not provided diff --git a/openfoia/templates.py b/openfoia/templates.py index 111771b..83b958f 100644 --- a/openfoia/templates.py +++ b/openfoia/templates.py @@ -57,7 +57,9 @@ def standard_request( This is the core template that works for most federal agencies. Uses language proven to be effective based on RCFP guidance. """ - date_str = datetime.now().strftime("%B %d, %Y") + # Local date is intended: this is the date printed on a letter the + # requester is sending, not a stored timestamp. + date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005 # Build date range clause if provided date_clause = "" @@ -243,7 +245,9 @@ def appeal_denial( Appeals must generally be filed within 90 days of the denial. """ - date_str = datetime.now().strftime("%B %d, %Y") + # Local date is intended: this is the date printed on a letter the + # requester is sending, not a stored timestamp. + date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005 request_date = original_request_date.strftime("%B %d, %Y") denial_date_str = denial_date.strftime("%B %d, %Y") @@ -348,7 +352,9 @@ def records_about_self( This template combines FOIA and Privacy Act requests for maximum coverage. """ - date_str = datetime.now().strftime("%B %d, %Y") + # Local date is intended: this is the date printed on a letter the + # requester is sending, not a stored timestamp. + date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005 letter = f"""{date_str} diff --git a/openfoia/tor_browse.py b/openfoia/tor_browse.py index 1fe4f9d..c3111eb 100644 --- a/openfoia/tor_browse.py +++ b/openfoia/tor_browse.py @@ -18,6 +18,7 @@ from rich import print as rprint +from .models import utcnow as _utcnow # --------------------------------------------------------------------------- # Constants @@ -185,14 +186,14 @@ async def browse( from datetime import datetime safe_name = re.sub(r"[^\w\-.]", "_", url.split("//", 1)[-1][:60]) - timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S") + timestamp = _utcnow().strftime("%Y%m%d_%H%M%S") filename = f"{timestamp}_{safe_name}.txt" out_path = save_dir / filename out_path.write_text( f"URL: {result['url']}\n" f"Title: {result['title']}\n" - f"Captured: {datetime.utcnow().isoformat()}Z\n" + f"Captured: {_utcnow().isoformat()}Z\n" f"Tor: {use_tor}\n" f"{'=' * 60}\n\n" f"{content}" diff --git a/tests/test_datetime_semantics.py b/tests/test_datetime_semantics.py new file mode 100644 index 0000000..a8c4156 --- /dev/null +++ b/tests/test_datetime_semantics.py @@ -0,0 +1,112 @@ +"""Regression tests for the naive-UTC datetime convention. + +`datetime.utcnow()` is deprecated (3.12+) and this project's CI targets 3.13, +so it has to go. But the ORM columns are `DateTime` **without** timezone, i.e. +naive UTC, and model helpers compare stored values against "now". Swapping in +an aware `datetime.now(timezone.utc)` would make those comparisons raise +`TypeError: can't subtract offset-naive and offset-aware datetimes`. + +These tests pin the convention: keep storing naive UTC, but stop calling the +deprecated API. Changing to fully timezone-aware storage is a separate, +deliberate migration. +""" + +from __future__ import annotations + +import warnings +from datetime import datetime, timedelta, timezone + + +def test_utcnow_helper_returns_naive_datetime(): + """Storage stays naive so it stays comparable with existing rows.""" + from openfoia.models import utcnow + + now = utcnow() + + assert now.tzinfo is None, "helper returned an aware datetime; ORM columns are naive" + + +def test_utcnow_helper_is_actually_utc(): + from openfoia.models import utcnow + + delta = abs((utcnow() - datetime.now(timezone.utc).replace(tzinfo=None)).total_seconds()) + + assert delta < 5, "helper is not UTC-based" + + +def test_utcnow_helper_emits_no_deprecation_warning(): + """The whole point: no deprecated utcnow() under the hood.""" + from openfoia.models import utcnow + + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + utcnow() + + +def test_days_pending_still_works_against_naive_storage(): + """This is what a blind aware-datetime conversion would break.""" + from openfoia.models import Request, utcnow + + req = Request() + req.sent_at = utcnow() - timedelta(days=3) + + assert req.days_pending() == 3 + + +def test_is_overdue_still_works_against_naive_storage(): + from openfoia.models import Request, utcnow + + req = Request() + req.due_date = utcnow() - timedelta(days=1) + assert req.is_overdue() is True + + req.due_date = utcnow() + timedelta(days=1) + assert req.is_overdue() is False + + +def test_no_local_time_written_into_utc_columns(): + """`sent_at = datetime.now()` wrote LOCAL time into a UTC column. + + Every reader (`days_pending`, `is_overdue`) compares those values against + UTC, so for a user in UTC+9 or UTC-8 the statutory FOIA deadline was off + by up to a day. Timestamps destined for storage must use the helper. + """ + import re + from pathlib import Path + + import openfoia + + pkg = Path(openfoia.__file__).parent + offenders = [] + # Assignments of a bare datetime.now() to a persisted timestamp column. + pattern = re.compile( + r"\.(sent_at|created_at|received_at|processed_at|acknowledged_at|" + r"completed_at|occurred_at|due_date|ends_at)\s*=\s*datetime\.now\(\s*\)" + ) + for path in pkg.rglob("*.py"): + for lineno, line in enumerate(path.read_text().splitlines(), 1): + if pattern.search(line): + offenders.append(f"{path.relative_to(pkg)}:{lineno}") + + assert offenders == [], f"local time written into a naive-UTC column: {offenders}" + + +def test_no_deprecated_utcnow_calls_remain_in_package(): + """`datetime.utcnow()` is deprecated and removed-in-future.""" + import re + from pathlib import Path + + import openfoia + + pkg = Path(openfoia.__file__).parent + offenders = [] + for path in pkg.rglob("*.py"): + for lineno, line in enumerate(path.read_text().splitlines(), 1): + if line.strip().startswith("#"): + continue + # Attribute-style calls only: `datetime.utcnow()` and aliased + # forms like `dt.utcnow()`. A bare `utcnow()` is our own helper. + if re.search(r"\w\.utcnow\s*\(", line): + offenders.append(f"{path.relative_to(pkg)}:{lineno}") + + assert offenders == [], f"deprecated datetime.utcnow() still called: {offenders}" From ec42ba9513d64ef47b1387615e875660a8aa9e10 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 13:11:42 +0000 Subject: [PATCH 2/4] Widen the ruff rule set and fix the findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the security review, which pinned ruff to four rules to stop an upgrade turning CI red. That restored determinism but left ~300 findings unreported, and some of them had real signal. This enables the rules that matter and fixes what they found, rather than leaving the narrow pin as the de-facto answer. Enabled: I, UP, B, SIM, DTZ, C4, PIE, RUF (on top of E4/E7/E9/F). Real defects found and fixed: - B023: the OCR loop's worker closure did not bind `image`. Benign today because it is awaited within the iteration, but parallelizing that loop — an obvious future optimization — would have silently OCR'd the last page N times. Now bound explicitly. - B904: 17 `raise` sites inside `except` blocks lost the exception chain. All are control-flow re-raises (typer.Exit, SystemExit, HTTPException) after a friendly message is printed, so they now say `from None` explicitly instead of dropping the chain by accident. - F401: three optional-dependency availability probes were indistinguishable from dead imports; annotated as deliberate. Judgement calls, documented in pyproject rather than applied blindly: - UP042 (str-Enum -> StrEnum) is ignored. It changes what `str(member)` returns, and these enums are persisted through SQLAlchemy and serialized into exports — a data-format change wearing a lint fix's clothing. - B008 is ignored for server.py/cli.py: `= Depends(...)` and `= typer.Option(...)` in signature defaults are the frameworks' idiom. - S101/SLF001 ignored under tests/ (asserts and private access are the point). - BLE001 (~82 blind excepts) stays out for now, with a note explaining why: it is high signal for this codebase but needs a per-site audit of error handling, which deserves its own reviewable change. Also: U+2028/U+2029 in the XSS escaper and its test are now written as `
` source escapes instead of raw invisible characters — same behaviour, but reviewers can actually see them. Everything else was mechanical: import ordering, modern typing syntax, contextlib.suppress, ClassVar on mutable class attributes, two collapsible ifs (both verified to have no else branch, so behaviour is unchanged). 181 tests pass; lint and format checks clean; `openfoia init` verified for both plain and encrypted databases. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KWzqFx1d2uB6cjQV81mgJh --- openfoia/agent.py | 16 +-- openfoia/cli.py | 124 ++++++++---------- openfoia/config.py | 8 +- openfoia/crossref.py | 2 - openfoia/db.py | 9 +- openfoia/ftm.py | 2 +- openfoia/gateways/__init__.py | 4 +- openfoia/gateways/email.py | 1 - openfoia/gateways/fax.py | 10 +- openfoia/gateways/mail.py | 11 +- openfoia/graph_template.py | 4 +- .../migrations/versions/001_initial_schema.py | 8 +- openfoia/models.py | 68 +++++----- openfoia/pipeline/__init__.py | 2 +- openfoia/pipeline/extract.py | 24 ++-- openfoia/pipeline/ingest.py | 1 - openfoia/pipeline/metadata.py | 4 +- openfoia/pipeline/ocr.py | 10 +- openfoia/pipeline/web.py | 28 ++-- openfoia/records/muckrock.py | 4 +- openfoia/security.py | 20 +-- openfoia/server.py | 10 +- openfoia/templates.py | 11 +- openfoia/tor_browse.py | 8 +- pyproject.toml | 58 ++++++-- tests/benchmark_extraction.py | 4 +- tests/test_datetime_semantics.py | 4 +- tests/test_extraction.py | 5 +- tests/test_security_gateways.py | 3 +- tests/test_security_injection.py | 6 +- 30 files changed, 232 insertions(+), 237 deletions(-) diff --git a/openfoia/agent.py b/openfoia/agent.py index b7e81fc..9c40531 100644 --- a/openfoia/agent.py +++ b/openfoia/agent.py @@ -12,9 +12,9 @@ from __future__ import annotations +import contextlib import logging from dataclasses import dataclass -from datetime import datetime from typing import Any from .models import Agency, Request, RequestStatus @@ -319,10 +319,8 @@ async def _search_agencies(self, params: dict[str, Any]) -> dict[str, Any]: if level and level != "all": from .models import AgencyLevel - try: + with contextlib.suppress(ValueError): agencies = agencies.filter(Agency.level == AgencyLevel(level)) - except ValueError: - pass results = agencies.limit(20).all() return { @@ -401,8 +399,6 @@ async def _draft_request(self, params: dict[str, Any]) -> dict[str, Any]: user = self.db.query(User).first() if user and agency: - from datetime import datetime as dt - req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}" new_req = Request( id=request_id, @@ -508,10 +504,8 @@ async def _list_requests(self, params: dict[str, Any]) -> dict[str, Any]: status_filter = params.get("status") if status_filter and status_filter != "all": - try: + with contextlib.suppress(ValueError): query = query.filter(Request.status == RequestStatus(status_filter)) - except ValueError: - pass agency_id = params.get("agency_id") if agency_id: @@ -642,10 +636,8 @@ async def _search_entities(self, params: dict[str, Any]) -> dict[str, Any]: if entity_type and entity_type != "all": from .models import EntityType - try: + with contextlib.suppress(ValueError): query = query.filter(Entity.entity_type == EntityType(entity_type)) - except ValueError: - pass results = query.limit(50).all() diff --git a/openfoia/cli.py b/openfoia/cli.py index 2b13069..c89978b 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -3,10 +3,11 @@ from __future__ import annotations import asyncio +import contextlib import json from datetime import datetime from pathlib import Path -from typing import Any, Optional +from typing import Any import typer from rich import print as rprint @@ -38,7 +39,7 @@ def init( duress: bool = typer.Option( False, "--duress", help="Also set up a decoy database (prompts for a passphrase)" ), - password: Optional[str] = typer.Option( + password: str | None = typer.Option( None, "--password", prompt=False, @@ -46,7 +47,7 @@ def init( help="Encryption passphrase. Prefer --encrypt, which prompts: a passphrase " "passed here is recorded in shell history and visible in the process list.", ), - duress_password: Optional[str] = typer.Option( + duress_password: str | None = typer.Option( None, "--duress-password", prompt=False, @@ -384,7 +385,7 @@ def guide(): def serve( port: int = typer.Option(0, "--port", "-p", help="Port to run on (0 = random)"), host: str = typer.Option("127.0.0.1", "--host", "-h", help="Host to bind to"), - browser: Optional[str] = typer.Option( + browser: str | None = typer.Option( None, "--browser", "-b", help="Browser to open (safari/firefox/chrome/brave/tor)" ), private: bool = typer.Option( @@ -577,7 +578,7 @@ def encrypt( encrypt_database(password) except Exception as e: rprint(f"[bold red]Encryption failed:[/bold red] {e}") - raise typer.Exit(1) + raise typer.Exit(1) from None rprint("[bold green]Database encrypted successfully.[/bold green]") rprint("[green]Plaintext database and its WAL/journal files were shredded in place.[/green]") @@ -680,8 +681,8 @@ def config( def request_new( agency: str = typer.Option(..., "--agency", "-a", help="Target agency name or ID"), subject: str = typer.Option(..., "--subject", "-s", help="Request subject"), - body: Optional[str] = typer.Option(None, "--body", "-b", help="Request body (or use --file)"), - body_file: Optional[Path] = typer.Option( + body: str | None = typer.Option(None, "--body", "-b", help="Request body (or use --file)"), + body_file: Path | None = typer.Option( None, "--file", "-f", help="File containing request body" ), method: str = typer.Option("email", "--method", "-m", help="Delivery method (email/fax/mail)"), @@ -783,8 +784,8 @@ def request_new( @request_app.command("list") def request_list( - status: Optional[str] = typer.Option(None, "--status", "-s", help="Filter by status"), - agency: Optional[str] = typer.Option(None, "--agency", "-a", help="Filter by agency"), + status: str | None = typer.Option(None, "--status", "-s", help="Filter by status"), + agency: str | None = typer.Option(None, "--agency", "-a", help="Filter by agency"), limit: int = typer.Option(20, "--limit", "-n", help="Maximum results"), ): """List FOIA requests.""" @@ -807,7 +808,7 @@ def request_list( query = query.filter(RequestModel.status == status_enum) except ValueError: rprint(f"[red]Invalid status '{status}'.[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None if agency: query = query.filter( @@ -940,17 +941,17 @@ def request_status( def request_send( agency: str = typer.Option(..., "--agency", "-a", help="Target agency (name or abbreviation)"), subject: str = typer.Option(..., "--subject", "-s", help="Request subject"), - body: Optional[str] = typer.Option(None, "--body", "-b", help="Request body text"), - body_file: Optional[Path] = typer.Option( + body: str | None = typer.Option(None, "--body", "-b", help="Request body text"), + body_file: Path | None = typer.Option( None, "--file", "-f", help="File containing request body" ), - template: Optional[str] = typer.Option( + template: str | None = typer.Option( None, "--template", "-t", help="Use template (standard/self)" ), name: str = typer.Option(..., "--name", "-n", help="Your full name"), email: str = typer.Option(..., "--email", "-e", help="Your email address"), method: str = typer.Option("email", "--method", "-m", help="Delivery method (email/fax/mail)"), - to_address: Optional[str] = typer.Option( + to_address: str | None = typer.Option( None, "--to", help="Override recipient address (email, fax number, or mailing address)" ), dry_run: bool = typer.Option( @@ -1105,10 +1106,8 @@ def request_send( config = {} if config_path.exists(): - try: + with contextlib.suppress(json.JSONDecodeError): config = json.loads(config_path.read_text()) - except json.JSONDecodeError: - pass # Build and send via the appropriate gateway if method == "email": @@ -1266,9 +1265,7 @@ def request_send( @docs_app.command("ingest") def docs_ingest( path: Path = typer.Argument(..., help="File or directory to ingest"), - request_id: Optional[str] = typer.Option( - None, "--request", "-r", help="Associate with request" - ), + request_id: str | None = typer.Option(None, "--request", "-r", help="Associate with request"), recursive: bool = typer.Option( True, "--recursive/--no-recursive", help="Recurse into directories" ), @@ -1464,7 +1461,7 @@ def docs_ocr( backend: str = typer.Option( "tesseract", "--backend", "-b", help="OCR backend (tesseract/google/aws)" ), - output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output text file"), + output: Path | None = typer.Option(None, "--output", "-o", help="Output text file"), ): """Run OCR on a PDF document. @@ -1506,10 +1503,10 @@ def docs_ocr( rprint(f"[red]Missing dependency: {e}[/red]") rprint("[dim]Install with: pip install pytesseract pdf2image[/dim]") rprint("[dim]Also need: brew install tesseract poppler (macOS)[/dim]") - raise typer.Exit(1) + raise typer.Exit(1) from None except Exception as e: rprint(f"[red]OCR failed: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None progress.update(task, description="Detecting redactions...") redactions = asyncio.run(detector.analyze(result.text, file_path)) @@ -1552,12 +1549,10 @@ def docs_ocr( @agency_app.command("list") def agency_list( - level: Optional[str] = typer.Option( + level: str | None = typer.Option( None, "--level", "-l", help="Filter by level (federal/state/local)" ), - state: Optional[str] = typer.Option( - None, "--state", "-s", help="Filter by state (2-letter code)" - ), + state: str | None = typer.Option(None, "--state", "-s", help="Filter by state (2-letter code)"), limit: int = typer.Option(50, "--limit", "-n", help="Maximum results"), ): """List agencies in the database.""" @@ -1578,7 +1573,7 @@ def agency_list( query = query.filter(Agency.level == level_enum) except ValueError: rprint(f"[red]Invalid level '{level}'. Use: federal, state, local, tribal[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None if state: query = query.filter(Agency.state == state.upper()) @@ -1746,9 +1741,9 @@ def template_generate( name: str = typer.Option(..., "--name", "-n", help="Your full name"), email: str = typer.Option(..., "--email", "-e", help="Your email address"), address: str = typer.Option("", "--address", help="Your mailing address"), - organization: Optional[str] = typer.Option(None, "--org", help="Your organization"), + organization: str | None = typer.Option(None, "--org", help="Your organization"), journalist: bool = typer.Option(False, "--journalist", "-j", help="You are a journalist"), - output: Optional[Path] = typer.Option( + output: Path | None = typer.Option( None, "--output", "-o", help="Output file (default: stdout)" ), no_fee_waiver: bool = typer.Option( @@ -2316,9 +2311,9 @@ def campaign_progress( @analyze_app.command("extract") def analyze_extract( document_id: str = typer.Argument(..., help="Document ID to analyze"), - output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output file"), + output: Path | None = typer.Option(None, "--output", "-o", help="Output file"), force: bool = typer.Option(False, "--force", help="Re-extract even if already done"), - model: Optional[str] = typer.Option( + model: str | None = typer.Option( None, "--model", "-m", help="LLM model (e.g. llama3.1:8b, llama3.2:3b)" ), ensemble: bool = typer.Option( @@ -2403,7 +2398,7 @@ def analyze_extract( except Exception as e: rprint(f"[red]Extraction failed: {e}[/red]") rprint("[dim]Ensure AI provider is configured: openfoia config --init[/dim]") - raise typer.Exit(1) + raise typer.Exit(1) from None if not result.entities: rprint("[yellow]No entities found in document.[/yellow]") @@ -2549,9 +2544,7 @@ def analyze_graphs_list(): size = f.stat().st_size size_str = f"{size / 1024:.0f}KB" if size > 1024 else f"{size}B" # Local time is intended: this is a file listing shown to the user. - modified = datetime.fromtimestamp( # noqa: DTZ006 - f.stat().st_mtime - ).strftime("%Y-%m-%d %H:%M") + modified = datetime.fromtimestamp(f.stat().st_mtime).strftime("%Y-%m-%d %H:%M") # noqa: DTZ006 table.add_row(name, " + ".join(types), size_str, modified) @@ -2562,13 +2555,11 @@ def analyze_graphs_list(): @analyze_app.command("graph") def analyze_graph( - request_id: Optional[str] = typer.Option( - None, "--request", "-r", help="Analyze single request" - ), - campaign_id: Optional[str] = typer.Option( + request_id: str | None = typer.Option(None, "--request", "-r", help="Analyze single request"), + campaign_id: str | None = typer.Option( None, "--campaign", "-c", help="Analyze entire campaign" ), - name: Optional[str] = typer.Option( + name: str | None = typer.Option( None, "--name", "-n", help="Save as named graph (stored in ~/.openfoia/graphs/)" ), output: Path = typer.Option( @@ -2743,10 +2734,7 @@ def _load_config_data() -> tuple[Path, dict]: from .db import get_data_dir config_path = get_data_dir() / "config.json" - if config_path.exists(): - data = json.loads(config_path.read_text()) - else: - data = {} + data = json.loads(config_path.read_text()) if config_path.exists() else {} return config_path, data @@ -2799,7 +2787,7 @@ def entities_add( re.compile(pattern) except re.error as e: rprint(f"[red]Invalid regex pattern: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None name = name.upper().replace(" ", "_") @@ -3251,7 +3239,7 @@ def entities_export( @entities_app.command("test") def entities_test( text: str = typer.Option(None, "--text", "-t", help="Test text (or reads from stdin)"), - file: Optional[Path] = typer.Option(None, "--file", "-f", help="Test against a file"), + file: Path | None = typer.Option(None, "--file", "-f", help="Test against a file"), ): """Test your custom entity types against sample text. @@ -3530,10 +3518,10 @@ def browse( ) ) except SystemExit: - raise typer.Exit(1) + raise typer.Exit(1) from None except Exception as e: rprint(f"[red]Browse failed:[/red] {e}") - raise typer.Exit(1) + raise typer.Exit(1) from None rprint(f"\n[cyan]Title:[/cyan] {result.get('title', 'N/A')}") rprint(f"[cyan]URL:[/cyan] {result.get('url', url)}") @@ -3657,9 +3645,7 @@ def purge( def ingest_url( url: str = typer.Option(..., "--url", "-u", help="URL to fetch and ingest"), tor: bool = typer.Option(False, "--tor", help="Route through Tor SOCKS5 proxy"), - output: Optional[Path] = typer.Option( - None, "--output", "-o", help="Save extracted text to file" - ), + output: Path | None = typer.Option(None, "--output", "-o", help="Save extracted text to file"), ): """Ingest a web page into the document pipeline. @@ -3694,7 +3680,7 @@ def ingest_url( rprint(f"[red]Failed to fetch URL: {e}[/red]") if tor: rprint("[dim]Make sure Tor is running: brew install tor && tor[/dim]") - raise typer.Exit(1) + raise typer.Exit(1) from None rprint("\n[bold green]Archived web page[/bold green]") rprint("=" * 50) @@ -3735,10 +3721,10 @@ def records_search( "-s", help="Data source (muckrock, opencorporates, sec)", ), - jurisdiction: Optional[str] = typer.Option( + jurisdiction: str | None = typer.Option( None, "--jurisdiction", "-j", help="Jurisdiction filter (e.g. us_ca, gb)" ), - filing_type: Optional[str] = typer.Option( + filing_type: str | None = typer.Option( None, "--type", "-t", help="Filing type filter for SEC (e.g. 10-K, 8-K)" ), limit: int = typer.Option(10, "--limit", "-n", help="Maximum results to display"), @@ -3789,7 +3775,7 @@ def records_search( result = asyncio.run(adapter.search(query, **kwargs)) except Exception as e: rprint(f"[red]Search failed: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None if raw: rprint( @@ -4035,7 +4021,7 @@ def records_fetch( result_id, text = asyncio.run(adapter.pull_text(doc_id)) except Exception as e: rprint(f"[red]Fetch failed: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None if not result_id or not text: rprint(f"[red]Could not fetch text for document {doc_id}.[/red]") @@ -4091,7 +4077,7 @@ def records_download( entity = asyncio.run(adapter.fetch(request_id)) except Exception as e: rprint(f"[red]Failed to fetch request: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None if not entity: rprint(f"[red]Request {request_id} not found on MuckRock.[/red]") @@ -4124,7 +4110,7 @@ def records_download( downloaded = asyncio.run(adapter.download_files(request_id, str(output))) except Exception as e: rprint(f"[red]Download failed: {e}[/red]") - raise typer.Exit(1) + raise typer.Exit(1) from None rprint(f"\n[green]{len(downloaded)} file(s) downloaded to {output}/[/green]") @@ -4216,22 +4202,22 @@ def records_download( @app.command() def crossref( - request_id: Optional[str] = typer.Option( + request_id: str | None = typer.Option( None, "--request", "-r", help="Cross-ref entities from a specific request" ), - document_id: Optional[str] = typer.Option( + document_id: str | None = typer.Option( None, "--document", "-d", help="Cross-ref entities from a specific document" ), - sources: Optional[str] = typer.Option( + sources: str | None = typer.Option( None, "--sources", help="Comma-separated sources (muckrock,opencorporates,sec,opensanctions,documentcloud)", ), - icij_data: Optional[Path] = typer.Option( + icij_data: Path | None = typer.Option( None, "--icij-data", help="Path to downloaded ICIJ CSV data" ), - output: Optional[Path] = typer.Option(None, "--output", "-o", help="Save report to file"), - ftm: Optional[Path] = typer.Option( + output: Path | None = typer.Option(None, "--output", "-o", help="Save report to file"), + ftm: Path | None = typer.Option( None, "--ftm", help="Export results as FollowTheMoney JSON-lines" ), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the network confirmation prompt"), @@ -4341,9 +4327,7 @@ def crossref( rprint("[bold]Cross-referencing entities...[/bold]") def _progress(event: str, msg: str) -> None: - if event == "start": - rprint(f"[dim] {msg}[/dim]") - elif event == "entity": + if event == "start" or event == "entity": rprint(f"[dim] {msg}[/dim]") report = asyncio.run( @@ -4435,7 +4419,7 @@ def _progress(event: str, msg: str) -> None: @analyze_app.command("export") def analyze_export( output: Path = typer.Option("entities.ftm.json", "--output", "-o", help="Output file path"), - request_id: Optional[str] = typer.Option( + request_id: str | None = typer.Option( None, "--request", "-r", help="Export from specific request" ), ): @@ -4508,7 +4492,7 @@ def analyze_export( @analyze_app.command("import") def analyze_import( file: Path = typer.Argument(..., help="FtM JSON-lines file to import"), - tag: Optional[str] = typer.Option(None, "--tag", "-t", help="Tag for this import batch"), + tag: str | None = typer.Option(None, "--tag", "-t", help="Tag for this import batch"), ): """Import entities from a FollowTheMoney JSON-lines file. diff --git a/openfoia/config.py b/openfoia/config.py index e2dc191..9771fb6 100644 --- a/openfoia/config.py +++ b/openfoia/config.py @@ -13,6 +13,7 @@ from __future__ import annotations +import contextlib import json import os from dataclasses import dataclass, field @@ -425,7 +426,6 @@ def save_config(config: OpenFOIAConfig, config_path: Path | str | None = None) - json.dump(data, f, indent=2) if os.name != "nt": - try: - os.chmod(path, 0o600) # tighten a pre-existing looser file - except OSError: - pass + # Tighten a pre-existing looser file; best-effort on odd filesystems. + with contextlib.suppress(OSError): + os.chmod(path, 0o600) diff --git a/openfoia/crossref.py b/openfoia/crossref.py index 38be96b..a808330 100644 --- a/openfoia/crossref.py +++ b/openfoia/crossref.py @@ -89,8 +89,6 @@ class _RateLimited(BaseException): in individual checkers don't swallow it — the crossref loop catches it. """ - pass - def _check_rate_limit(result: Any) -> None: """Raise _RateLimited if the search result indicates a rate limit error.""" diff --git a/openfoia/db.py b/openfoia/db.py index 235d9d5..12140b3 100644 --- a/openfoia/db.py +++ b/openfoia/db.py @@ -12,9 +12,10 @@ import sqlite3 import stat import tempfile -from contextlib import contextmanager +from collections.abc import Generator +from contextlib import contextmanager, suppress from pathlib import Path -from typing import Any, Generator +from typing import Any from sqlalchemy import create_engine, event from sqlalchemy.engine import Engine @@ -335,10 +336,8 @@ def _restrict_db_permissions(db_path: Path) -> None: return for candidate in (db_path, *(Path(str(db_path) + s) for s in _DB_SIDECAR_SUFFIXES)): if candidate.is_file(): - try: + with suppress(OSError): os.chmod(candidate, 0o600) - except OSError: - pass def encrypt_database(password: str) -> None: diff --git a/openfoia/ftm.py b/openfoia/ftm.py index 622f1a4..6fb66d0 100644 --- a/openfoia/ftm.py +++ b/openfoia/ftm.py @@ -66,7 +66,7 @@ def _try_ftm_available() -> bool: """Check if followthemoney library is installed.""" try: - import followthemoney # noqa: F401 + import followthemoney # noqa: F401 - availability probe for the optional extra return True except ImportError: diff --git a/openfoia/gateways/__init__.py b/openfoia/gateways/__init__.py index bdf570c..57e72e5 100644 --- a/openfoia/gateways/__init__.py +++ b/openfoia/gateways/__init__.py @@ -15,7 +15,7 @@ __all__ = [ "DeliveryGateway", "DeliveryResult", - "TwilioFaxGateway", - "LobMailGateway", "EmailGateway", + "LobMailGateway", + "TwilioFaxGateway", ] diff --git a/openfoia/gateways/email.py b/openfoia/gateways/email.py index cce6749..f998ad6 100644 --- a/openfoia/gateways/email.py +++ b/openfoia/gateways/email.py @@ -5,7 +5,6 @@ import asyncio import smtplib import ssl -from datetime import datetime from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText diff --git a/openfoia/gateways/fax.py b/openfoia/gateways/fax.py index 41e459e..9a14897 100644 --- a/openfoia/gateways/fax.py +++ b/openfoia/gateways/fax.py @@ -16,7 +16,7 @@ import io import logging import os -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any from uuid import uuid4 @@ -138,7 +138,7 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult: return DeliveryResult( status=DeliveryStatus.PENDING, reference_id=fax.sid, - sent_at=datetime.now(timezone.utc), + sent_at=datetime.now(UTC), cost_cents=pages * self.COST_PER_PAGE_CENTS, metadata={ "to": payload.recipient_address, @@ -311,9 +311,7 @@ def _generate_pdf_reportlab(self, payload: DeliveryPayload) -> bytes: story.append(Paragraph(f"TO: {_esc(payload.recipient_name)}", body_style)) story.append(Paragraph(f"FAX: {_esc(payload.recipient_address)}", body_style)) story.append( - Paragraph( - f"DATE: {datetime.now(timezone.utc).strftime('%B %d, %Y')}", body_style - ) + Paragraph(f"DATE: {datetime.now(UTC).strftime('%B %d, %Y')}", body_style) ) story.append( Paragraph(f"RE: FOIA Request - {_esc(payload.subject)}", body_style) @@ -392,7 +390,7 @@ def _generate_pdf_minimal(self, payload: DeliveryPayload) -> bytes: This is a fallback that creates a bare-bones but valid PDF. """ - date_str = datetime.now(timezone.utc).strftime("%B %d, %Y") + date_str = datetime.now(UTC).strftime("%B %d, %Y") sender = (payload.return_address or "[Requester Name]").split("\n")[0] text_lines = [ diff --git a/openfoia/gateways/mail.py b/openfoia/gateways/mail.py index 1ab98f1..81bfd47 100644 --- a/openfoia/gateways/mail.py +++ b/openfoia/gateways/mail.py @@ -14,7 +14,7 @@ import html import logging import re -from datetime import datetime, timezone +from datetime import UTC, datetime from typing import Any from .base import DeliveryGateway, DeliveryPayload, DeliveryResult, DeliveryStatus @@ -94,7 +94,8 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult: sends it via Lob's print-and-mail API, and returns tracking info. """ try: - import lob # noqa: F401 — triggers ImportError if not installed + # Availability probe: raises ImportError if the extra is missing. + import lob # noqa: F401 lob_client = self._get_lob() @@ -142,7 +143,7 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult: return DeliveryResult( status=DeliveryStatus.SENT, reference_id=letter.id, - sent_at=datetime.now(timezone.utc), + sent_at=datetime.now(UTC), cost_cents=self.estimate_cost(payload), metadata={ "tracking_number": tracking_number, @@ -274,7 +275,7 @@ def _estimate_pages(self, payload: DeliveryPayload) -> int: pages = max(1, len(payload.body) // 3000 + 1) if payload.attachments: - for filename, content in payload.attachments: + for _filename, content in payload.attachments: pages += max(1, len(content) // 3000 + 1) return pages @@ -353,7 +354,7 @@ def _generate_letter_html(self, payload: DeliveryPayload) -> str: Lob renders HTML to PDF for printing. The template uses standard fonts and margins suitable for USPS mailing. """ - date_str = datetime.now(timezone.utc).strftime("%B %d, %Y") + date_str = datetime.now(UTC).strftime("%B %d, %Y") sender_name = self.return_address.get("name", "[Requester Name]") # Build sender address block for letterhead diff --git a/openfoia/graph_template.py b/openfoia/graph_template.py index 684bd3c..3c25a1c 100644 --- a/openfoia/graph_template.py +++ b/openfoia/graph_template.py @@ -32,8 +32,8 @@ def escape_json_for_script(graph_json: str) -> str: graph_json.replace("&", "\\u0026") .replace("<", "\\u003c") .replace(">", "\\u003e") - .replace("
", "\\u2028") - .replace("
", "\\u2029") + .replace("\u2028", "\\u2028") + .replace("\u2029", "\\u2029") ) diff --git a/openfoia/migrations/versions/001_initial_schema.py b/openfoia/migrations/versions/001_initial_schema.py index f61f6e3..0b8931f 100644 --- a/openfoia/migrations/versions/001_initial_schema.py +++ b/openfoia/migrations/versions/001_initial_schema.py @@ -6,16 +6,16 @@ """ -from typing import Sequence, Union +from collections.abc import Sequence import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. revision: str = "001" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: diff --git a/openfoia/models.py b/openfoia/models.py index 26a9c3e..d7c3539 100644 --- a/openfoia/models.py +++ b/openfoia/models.py @@ -11,26 +11,7 @@ from __future__ import annotations import enum -from datetime import datetime, timezone - - -def utcnow() -> datetime: - """Current UTC time as a **naive** datetime. - - The ORM's ``DateTime`` columns are timezone-naive, and model helpers such - as ``Request.days_pending`` compare stored values against "now". Returning - an aware datetime here would raise - ``TypeError: can't subtract offset-naive and offset-aware datetimes`` - against every existing row. - - This replaces the deprecated ``utcnow()`` (removed in a future - Python; CI already targets 3.13) while preserving the naive-UTC storage - convention exactly. Moving to timezone-aware columns is a separate, - deliberate migration. - """ - return datetime.now(timezone.utc).replace(tzinfo=None) - - +from datetime import UTC, datetime from typing import Any from uuid import uuid4 @@ -50,6 +31,23 @@ def utcnow() -> datetime: from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +def utcnow() -> datetime: + """Current UTC time as a **naive** datetime. + + The ORM's ``DateTime`` columns are timezone-naive, and model helpers such + as ``Request.days_pending`` compare stored values against "now". Returning + an aware datetime here would raise + ``TypeError: can't subtract offset-naive and offset-aware datetimes`` + against every existing row. + + This replaces the deprecated ``utcnow()`` (removed in a future + Python; CI already targets 3.13) while preserving the naive-UTC storage + convention exactly. Moving to timezone-aware columns is a separate, + deliberate migration. + """ + return datetime.now(UTC).replace(tzinfo=None) + + class Base(DeclarativeBase): pass @@ -157,8 +155,8 @@ class User(Base): created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow) # Relationships - requests: Mapped[list["Request"]] = relationship(back_populates="requester") - campaigns: Mapped[list["Campaign"]] = relationship( + requests: Mapped[list[Request]] = relationship(back_populates="requester") + campaigns: Mapped[list[Campaign]] = relationship( secondary=campaign_participants, back_populates="participants" ) @@ -193,7 +191,7 @@ class Agency(Base): total_requests_tracked: Mapped[int] = mapped_column(Integer, default=0) # Relationships - requests: Mapped[list["Request"]] = relationship(back_populates="agency") + requests: Mapped[list[Request]] = relationship(back_populates="agency") class Request(Base): @@ -243,11 +241,11 @@ class Request(Base): extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) # Relationships - requester: Mapped["User"] = relationship(back_populates="requests") - agency: Mapped["Agency"] = relationship(back_populates="requests") - campaign: Mapped["Campaign | None"] = relationship(back_populates="requests") - documents: Mapped[list["Document"]] = relationship(back_populates="request") - timeline: Mapped[list["TimelineEvent"]] = relationship(back_populates="request") + requester: Mapped[User] = relationship(back_populates="requests") + agency: Mapped[Agency] = relationship(back_populates="requests") + campaign: Mapped[Campaign | None] = relationship(back_populates="requests") + documents: Mapped[list[Document]] = relationship(back_populates="request") + timeline: Mapped[list[TimelineEvent]] = relationship(back_populates="request") def days_pending(self) -> int: """Days since request was sent.""" @@ -294,8 +292,8 @@ class Document(Base): processed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) # Relationships - request: Mapped["Request"] = relationship(back_populates="documents") - entities: Mapped[list["Entity"]] = relationship(back_populates="source_document") + request: Mapped[Request] = relationship(back_populates="documents") + entities: Mapped[list[Entity]] = relationship(back_populates="source_document") class Entity(Base): @@ -323,8 +321,8 @@ class Entity(Base): extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) # Relationships - source_document: Mapped["Document"] = relationship(back_populates="entities") - linked_entities: Mapped[list["Entity"]] = relationship( + source_document: Mapped[Document] = relationship(back_populates="entities") + linked_entities: Mapped[list[Entity]] = relationship( secondary=entity_links, primaryjoin=id == entity_links.c.source_id, secondaryjoin=id == entity_links.c.target_id, @@ -356,10 +354,10 @@ class Campaign(Base): ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) # Relationships - participants: Mapped[list["User"]] = relationship( + participants: Mapped[list[User]] = relationship( secondary=campaign_participants, back_populates="campaigns" ) - requests: Mapped[list["Request"]] = relationship(back_populates="campaign") + requests: Mapped[list[Request]] = relationship(back_populates="campaign") def request_count(self) -> int: return len(self.requests) @@ -387,7 +385,7 @@ class TimelineEvent(Base): extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True) # Relationships - request: Mapped["Request"] = relationship(back_populates="timeline") + request: Mapped[Request] = relationship(back_populates="timeline") # === Database Setup === diff --git a/openfoia/pipeline/__init__.py b/openfoia/pipeline/__init__.py index 07a1163..b76078f 100644 --- a/openfoia/pipeline/__init__.py +++ b/openfoia/pipeline/__init__.py @@ -14,6 +14,6 @@ __all__ = [ "DocumentIngester", - "OCREngine", "EntityExtractor", + "OCREngine", ] diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py index f4e723f..18b1eff 100644 --- a/openfoia/pipeline/extract.py +++ b/openfoia/pipeline/extract.py @@ -210,7 +210,7 @@ def _is_protected_acronym(raw: str) -> bool: return s in _KEEP_ACRONYMS or (s.isupper() and 2 <= len(s) <= 6 and s.isalpha()) -def _mention_score(m: "Mention") -> float: +def _mention_score(m: Mention) -> float: """Score a mention for quality. 0.0 = definitely junk, should be dropped.""" s = _surface_clean(m.normalized_text) k = s.lower().rstrip(".") @@ -271,7 +271,7 @@ def _ocr_fold(text: str) -> str: return re.sub(r"[^a-z0-9 ]+", "", t) -def _core_tokens(text: str, etype: "EntityType") -> list[str]: +def _core_tokens(text: str, etype: EntityType) -> list[str]: """Extract meaningful tokens, stripping org suffixes.""" toks = re.findall(r"[a-z0-9]+", _ocr_fold(text)) if etype == EntityType.ORGANIZATION: @@ -414,7 +414,7 @@ def _get_gliner(): def _gliner_available() -> bool: """Check if GLiNER is installed.""" try: - import gliner # noqa: F401 + import gliner # noqa: F401 - availability probe for the optional extra return True except ImportError: @@ -1835,10 +1835,11 @@ def _group_key(m: Mention) -> tuple[str, str]: a, b = c1.canonical_text, c2.canonical_text # Substring match - if a.lower() in b.lower() or b.lower() in a.lower(): - should_merge = True - # OCR-fold similarity - elif SequenceMatcher(None, _ocr_fold(a), _ocr_fold(b)).ratio() >= 0.90: + if ( + a.lower() in b.lower() + or b.lower() in a.lower() + or SequenceMatcher(None, _ocr_fold(a), _ocr_fold(b)).ratio() >= 0.90 + ): should_merge = True # Token Jaccard for orgs elif c1.entity_type == EntityType.ORGANIZATION: @@ -2020,10 +2021,11 @@ def _find_or_create_canonical(self, entity: ExtractedEntity) -> str: return can_id can_norm = canonical["normalized"].lower() - if normalized in can_norm or can_norm in normalized: - if len(normalized) > 3 and len(can_norm) > 3: - canonical["aliases"].add(entity.raw_text) - return can_id + if (normalized in can_norm or can_norm in normalized) and ( + len(normalized) > 3 and len(can_norm) > 3 + ): + canonical["aliases"].add(entity.raw_text) + return can_id import uuid diff --git a/openfoia/pipeline/ingest.py b/openfoia/pipeline/ingest.py index beac585..6ebfc66 100644 --- a/openfoia/pipeline/ingest.py +++ b/openfoia/pipeline/ingest.py @@ -7,7 +7,6 @@ import mimetypes import shutil from dataclasses import dataclass -from datetime import datetime from pathlib import Path from typing import Any from uuid import uuid4 diff --git a/openfoia/pipeline/metadata.py b/openfoia/pipeline/metadata.py index c24e4a6..e99a216 100644 --- a/openfoia/pipeline/metadata.py +++ b/openfoia/pipeline/metadata.py @@ -252,9 +252,7 @@ def _strip_docx_metadata(file_path: Path, keep_hash: bool) -> dict[str, Any]: # Clear sensitive properties for attr in _DOCX_SENSITIVE_ATTRS: try: - if attr in ("created", "modified"): - setattr(core, attr, None) - elif attr == "revision": + if attr in ("created", "modified") or attr == "revision": setattr(core, attr, None) else: setattr(core, attr, "") diff --git a/openfoia/pipeline/ocr.py b/openfoia/pipeline/ocr.py index e232358..6b35d4a 100644 --- a/openfoia/pipeline/ocr.py +++ b/openfoia/pipeline/ocr.py @@ -6,7 +6,7 @@ import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any +from typing import Any, ClassVar #: Hard cap on pages rasterized in one OCR run. A hostile "FOIA response" #: declaring tens of thousands of pages would otherwise exhaust memory/disk @@ -147,7 +147,11 @@ def _convert(): for i, image in enumerate(images): # Get detailed OCR data - def _ocr(): + # `image` is bound as a default so the closure captures THIS + # iteration's page. It is awaited immediately today, so late + # binding does not bite — but parallelizing this loop later would + # otherwise silently OCR the last page N times. + def _ocr(image=image): data = pytesseract.image_to_data(image, output_type=pytesseract.Output.DICT) text = pytesseract.image_to_string(image) return data, text @@ -311,7 +315,7 @@ class RedactionDetector: """Detect and analyze redactions in documents.""" # Common FOIA exemption patterns - EXEMPTION_PATTERNS = { + EXEMPTION_PATTERNS: ClassVar[dict] = { r"\(b\)\(1\)": "National security", r"\(b\)\(2\)": "Internal personnel rules", r"\(b\)\(3\)": "Statutory exemption", diff --git a/openfoia/pipeline/web.py b/openfoia/pipeline/web.py index 5e39069..3fefe7f 100644 --- a/openfoia/pipeline/web.py +++ b/openfoia/pipeline/web.py @@ -6,10 +6,11 @@ from __future__ import annotations +import contextlib import hashlib import re from dataclasses import dataclass -from datetime import datetime, timezone +from datetime import UTC, datetime from html.parser import HTMLParser from pathlib import Path from typing import Any @@ -119,10 +120,9 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self._in_article += 1 elif tag == "main": self._in_main += 1 - elif tag == "div": - if self._current_div_text is None: - self._current_div_text = [] - self._div_depth = len(self._tag_stack) + elif tag == "div" and self._current_div_text is None: + self._current_div_text = [] + self._div_depth = len(self._tag_stack) def handle_endtag(self, tag: str) -> None: tag = tag.lower() @@ -140,10 +140,13 @@ def handle_endtag(self, tag: str) -> None: self._in_article -= 1 elif tag == "main" and self._in_main > 0: self._in_main -= 1 - elif tag == "div" and self._current_div_text is not None: - if len(self._tag_stack) <= self._div_depth: - self._div_blocks.append(self._current_div_text) - self._current_div_text = None + elif ( + tag == "div" + and self._current_div_text is not None + and len(self._tag_stack) <= self._div_depth + ): + self._div_blocks.append(self._current_div_text) + self._current_div_text = None if self._tag_stack and self._tag_stack[-1] == tag: self._tag_stack.pop() @@ -252,10 +255,9 @@ def _extract_content(html: str) -> tuple[str, str]: Returns (text_content, page_title). """ extractor = _ContentExtractor() - try: + # Best-effort parsing: malformed markup must not abort the archive. + with contextlib.suppress(Exception): extractor.feed(html) - except Exception: - pass # Best-effort parsing return extractor.get_content(), extractor.title.strip() @@ -292,7 +294,7 @@ async def fetch_url(url: str, use_tor: bool = False) -> WebFetchResult: # Extract main content text, title = _extract_content(cleaned_html) - fetched_at = datetime.now(timezone.utc).isoformat() + fetched_at = datetime.now(UTC).isoformat() return WebFetchResult( url=url, diff --git a/openfoia/records/muckrock.py b/openfoia/records/muckrock.py index c56a41f..a82f699 100644 --- a/openfoia/records/muckrock.py +++ b/openfoia/records/muckrock.py @@ -11,7 +11,7 @@ from __future__ import annotations import logging -from typing import Any +from typing import Any, ClassVar import httpx @@ -179,7 +179,7 @@ async def search( ) # Common abbreviation → full agency name mapping - _AGENCY_NAMES: dict[str, str] = { + _AGENCY_NAMES: ClassVar[dict[str, str]] = { "fbi": "Federal Bureau of Investigation", "cia": "Central Intelligence Agency", "nsa": "National Security Agency", diff --git a/openfoia/security.py b/openfoia/security.py index cb5dd3b..39d37d8 100644 --- a/openfoia/security.py +++ b/openfoia/security.py @@ -13,6 +13,7 @@ from __future__ import annotations +import contextlib import os import shutil import tempfile @@ -77,16 +78,13 @@ def secure_delete_dir(path: Path | str) -> int: secure_delete(item) count += 1 elif item.is_dir(): - try: + # Non-empty dir — will retry after its children are removed. + with contextlib.suppress(OSError): item.rmdir() - except OSError: - pass # non-empty dir — will retry after children removed # Remove the root directory itself - try: + with contextlib.suppress(OSError): path.rmdir() - except OSError: - pass return count @@ -172,16 +170,12 @@ def fill_free_space(path: Path | str, chunk_size_mb: int = 100) -> None: # Clean up fill files for fp in fill_files: - try: + with contextlib.suppress(OSError): fp.unlink() - except OSError: - pass # Try to remove the directory if we created it - try: + with contextlib.suppress(OSError): path.rmdir() - except OSError: - pass # --------------------------------------------------------------------------- @@ -400,7 +394,7 @@ def seed_decoy_db(db_path: Path, password: str | None = None) -> None: If password is provided, the decoy is encrypted with SQLCipher. """ import random - from datetime import datetime, timedelta + from datetime import timedelta from uuid import uuid4 from sqlalchemy import create_engine diff --git a/openfoia/server.py b/openfoia/server.py index 4bf8b6c..6ad888f 100644 --- a/openfoia/server.py +++ b/openfoia/server.py @@ -6,8 +6,8 @@ from __future__ import annotations +import contextlib import secrets -from datetime import datetime from pathlib import Path from uuid import uuid4 @@ -242,7 +242,7 @@ async def list_requests( raise HTTPException( status_code=400, detail=f"Invalid status '{status}'. Valid: {', '.join(s.value for s in RequestStatus)}", - ) + ) from None requests = query.order_by(FOIARequest.created_at.desc()).limit(limit).all() @@ -432,7 +432,7 @@ async def upload_document( raise HTTPException( status_code=413, detail=f"File exceeds the {MAX_UPLOAD_BYTES // (1024 * 1024)} MiB upload limit.", - ) + ) from None except Exception: dest.unlink(missing_ok=True) raise @@ -445,10 +445,8 @@ async def upload_document( ingester = DocumentIngester(storage_path=docs_dir) extracted_text = None - try: + with contextlib.suppress(Exception): extracted_text = await ingester._extract_text(dest, mime) - except Exception: - pass from .db import get_session from .models import Document, DocumentType diff --git a/openfoia/templates.py b/openfoia/templates.py index 83b958f..1769201 100644 --- a/openfoia/templates.py +++ b/openfoia/templates.py @@ -12,7 +12,6 @@ from dataclasses import dataclass from datetime import datetime -from typing import Optional @dataclass @@ -20,13 +19,13 @@ class RequesterInfo: """Information about the person filing the request.""" name: str - organization: Optional[str] = None + organization: str | None = None address: str = "" email: str = "" phone: str = "" is_journalist: bool = False is_educational: bool = False - publication: Optional[str] = None + publication: str | None = None @dataclass @@ -35,10 +34,10 @@ class RequestDetails: subject: str description: str - date_range_start: Optional[datetime] = None - date_range_end: Optional[datetime] = None + date_range_start: datetime | None = None + date_range_end: datetime | None = None keywords: list[str] = None - exclusions: Optional[str] = None + exclusions: str | None = None # === Base Request Template === diff --git a/openfoia/tor_browse.py b/openfoia/tor_browse.py index c3111eb..794bbe4 100644 --- a/openfoia/tor_browse.py +++ b/openfoia/tor_browse.py @@ -13,6 +13,7 @@ from __future__ import annotations +import contextlib from pathlib import Path from typing import Any @@ -83,7 +84,7 @@ async def browse( "[yellow]Run: openfoia install-extras browser[/yellow]\n" "[dim]Then: playwright install chromium[/dim]" ) - raise SystemExit(1) + raise SystemExit(1) from None if use_tor: rprint(_TOR_WARNING) @@ -183,7 +184,6 @@ async def browse( # Sanitize filename from URL import re - from datetime import datetime safe_name = re.sub(r"[^\w\-.]", "_", url.split("//", 1)[-1][:60]) timestamp = _utcnow().strftime("%Y%m%d_%H%M%S") @@ -205,10 +205,8 @@ async def browse( rprint( "\n[dim]Browser is open. Close the browser window or press Ctrl+C to exit.[/dim]" ) - try: + with contextlib.suppress(KeyboardInterrupt): await page.wait_for_event("close", timeout=0) - except KeyboardInterrupt: - pass await browser.close() diff --git a/pyproject.toml b/pyproject.toml index 4ccc489..bc9052f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,21 +76,51 @@ line-length = 100 target-version = "py311" [tool.ruff.lint] -# Pin the rule set explicitly. CI installs an unpinned `ruff>=0.1.0`, so -# relying on ruff's *defaults* means a new ruff release can silently widen -# the rule set and turn CI red on unchanged code — which is what happened. +# The rule set is pinned explicitly, never inherited from ruff's defaults: +# CI installs an unpinned `ruff>=0.1.0`, so relying on defaults lets a new +# ruff release silently change what CI enforces (which is exactly how this +# repo went from green to 325 errors on unchanged code). # -# This list is a FLOOR, not a target. It reproduces the rules the project was -# already passing, so lint is deterministic again; it is not a judgement that -# the wider rules are worthless. Several of them have real signal for this -# codebase — notably DTZ (naive/`utcnow()` datetimes, deprecated on the 3.13 -# CI target) and BLE001/S110 (blind excepts, the exact shape of the bug where -# a failed API lookup was reported as a clean result). -# -# Widening this set and fixing the ~300 resulting findings is tracked as -# follow-up work; it is a repo-wide diff that does not belong in a security -# change. Widen deliberately here — never by upgrading ruff. -select = ["E4", "E7", "E9", "F"] +# Widen this list deliberately, and fix the findings — do not narrow it to +# make a build pass. +select = [ + "E4", "E7", "E9", # pycodestyle (the historical default subset) + "F", # pyflakes + "I", # import sorting + "UP", # pyupgrade + "B", # bugbear + "SIM", # simplify + "DTZ", # naive datetimes — caught a real deadline bug + "C4", # comprehensions + "PIE", + "RUF", +] + +ignore = [ + # `class X(str, Enum)` -> `StrEnum` changes what `str(X.MEMBER)` returns + # ("federal" instead of "AgencyLevel.FEDERAL"). These enums are persisted + # through SQLAlchemy and serialized into API responses and exports, so + # that is a data-format change wearing a lint fix's clothing. Worth doing + # deliberately with migration checks, not as part of a lint sweep. + "UP042", +] + +# Not enabled yet, tracked as follow-up: +# BLE001 (~82) — blind `except Exception`. High signal for this codebase: +# a swallowed exception is what made a failed records lookup report as a +# clean result. Fixing it needs a per-site audit of error handling, which +# is its own reviewable change rather than a bulk sweep. +# S (bandit) — mostly subprocess/randomness false positives here, plus +# S101 which is just "asserts in tests". + +[tool.ruff.lint.per-file-ignores] +# FastAPI's dependency-injection idiom is `arg = Depends(...)` in the +# signature default; B008 flags every route. +"openfoia/server.py" = ["B008"] +# Typer's CLI idiom is the same: `arg: T = typer.Option(...)`. +"openfoia/cli.py" = ["B008"] +# Tests legitimately use `assert`, private access, and broad `raises`. +"tests/*" = ["S101", "SLF001"] [tool.mypy] python_version = "3.11" diff --git a/tests/benchmark_extraction.py b/tests/benchmark_extraction.py index 21c0fe6..2ce7846 100644 --- a/tests/benchmark_extraction.py +++ b/tests/benchmark_extraction.py @@ -15,12 +15,12 @@ import time from pathlib import Path +from openfoia.config import load_config from openfoia.pipeline.extract import ( EntityExtractor, _gliner_available, _llm_available, ) -from openfoia.config import load_config # Richer test doc with more entities and relationships DOC = """ @@ -312,7 +312,7 @@ def main(): p_found, p_total, p_missed = check_recall(result, EXPECTED_PERSONS, "person") o_found, o_total, o_missed = check_recall(result, EXPECTED_ORGS, "organization") - m_found, m_total, m_missed = check_recall(result, EXPECTED_MONEY, "money") + m_found, m_total, _m_missed = check_recall(result, EXPECTED_MONEY, "money") r_found, r_total, r_missed = check_relationship_recall(result, EXPECTED_RELATIONSHIPS) pr = p_found / p_total if p_total else 0 diff --git a/tests/test_datetime_semantics.py b/tests/test_datetime_semantics.py index a8c4156..b263a00 100644 --- a/tests/test_datetime_semantics.py +++ b/tests/test_datetime_semantics.py @@ -14,7 +14,7 @@ from __future__ import annotations import warnings -from datetime import datetime, timedelta, timezone +from datetime import UTC, datetime, timedelta def test_utcnow_helper_returns_naive_datetime(): @@ -29,7 +29,7 @@ def test_utcnow_helper_returns_naive_datetime(): def test_utcnow_helper_is_actually_utc(): from openfoia.models import utcnow - delta = abs((utcnow() - datetime.now(timezone.utc).replace(tzinfo=None)).total_seconds()) + delta = abs((utcnow() - datetime.now(UTC).replace(tzinfo=None)).total_seconds()) assert delta < 5, "helper is not UTC-based" diff --git a/tests/test_extraction.py b/tests/test_extraction.py index 0adf11c..7546e10 100644 --- a/tests/test_extraction.py +++ b/tests/test_extraction.py @@ -6,16 +6,17 @@ """ import asyncio + import pytest + from openfoia.pipeline.extract import ( EntityExtractor, ExtractionResult, _gliner_available, - _spacy_available, _llm_available, + _spacy_available, ) - # --------------------------------------------------------------------------- # Test documents with ground truth # --------------------------------------------------------------------------- diff --git a/tests/test_security_gateways.py b/tests/test_security_gateways.py index 5f37ae3..02742d5 100644 --- a/tests/test_security_gateways.py +++ b/tests/test_security_gateways.py @@ -9,6 +9,7 @@ import asyncio import os +import re import stat import pytest @@ -252,7 +253,7 @@ def stream(self, method, url): return _Stream(_Resp()) dest = tmp_path / "out.bin" - with pytest.raises(ValueError, match="(?i)size|large|bytes"): + with pytest.raises(ValueError, match=re.compile("size|large|bytes", re.I)): asyncio.run(download_to_file(_Client(), "https://x.test/a.pdf", dest, max_bytes=8 * 1024)) # It must have stopped early, not consumed the whole stream. diff --git a/tests/test_security_injection.py b/tests/test_security_injection.py index 45a7ae6..a4dd225 100644 --- a/tests/test_security_injection.py +++ b/tests/test_security_injection.py @@ -67,10 +67,10 @@ def test_graph_render_escapes_line_separators(tmp_path): """U+2028/U+2029 are valid JSON but break JS string literals.""" html = _render_graph( tmp_path, - {"nodes": [], "links": [], "documents": [{"id": "d1", "text": "a
b
c"}]}, + {"nodes": [], "links": [], "documents": [{"id": "d1", "text": "a\u2028b\u2029c"}]}, ) - assert "
" not in html - assert "
" not in html + assert "\u2028" not in html + assert "\u2029" not in html def test_graph_data_still_round_trips(tmp_path): From 3ee8fff6d47ec32eabcd3aeaba7f8ae9c4a12e47 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 13:13:03 +0000 Subject: [PATCH 3/4] Run CI on every pull request, not only those targeting main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pull_request: branches: [main]` meant a stacked PR — one branch based on another rather than on main — triggered no test jobs at all. It showed only the Copilot reviewer and otherwise looked reviewed and green while nothing had executed. Found because this PR is itself stacked. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KWzqFx1d2uB6cjQV81mgJh --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3780726..41583cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,11 @@ name: CI on: push: branches: [main] + # Run on every pull request, not just those targeting main. Restricting + # this to `branches: [main]` meant a stacked PR (one branch based on + # another) got no test run at all — it looked reviewed and green when + # nothing had actually executed. pull_request: - branches: [main] jobs: test: From c9a93839541e7413740797b5715e6db8c43cf030 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 13:16:57 +0000 Subject: [PATCH 4/4] Fix type annotations flagged by mypy in templates.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `RequestDetails.keywords` was annotated `list[str]` but defaulted to None, so the annotation lied to every caller. The field is currently unused, so there is no live bug, but mypy flagged it and the dataclass's other optional fields already use `| None = None` — matched that rather than switching to a default_factory, which would change the contract. Also give `list_templates()` a concrete `dict[str, str]` return type. mypy is clean on this file now (it runs in CI as continue-on-error). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KWzqFx1d2uB6cjQV81mgJh --- openfoia/templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfoia/templates.py b/openfoia/templates.py index 1769201..f7c1043 100644 --- a/openfoia/templates.py +++ b/openfoia/templates.py @@ -36,7 +36,7 @@ class RequestDetails: description: str date_range_start: datetime | None = None date_range_end: datetime | None = None - keywords: list[str] = None + keywords: list[str] | None = None exclusions: str | None = None @@ -417,7 +417,7 @@ def records_about_self( # === CLI Integration === -def list_templates() -> list[dict]: +def list_templates() -> list[dict[str, str]]: """Return list of available templates for CLI display.""" return [ {