diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3780726..41583cb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -3,8 +3,11 @@ name: CI
on:
push:
branches: [main]
+ # Run on every pull request, not just those targeting main. Restricting
+ # this to `branches: [main]` meant a stacked PR (one branch based on
+ # another) got no test run at all — it looked reviewed and green when
+ # nothing had actually executed.
pull_request:
- branches: [main]
jobs:
test:
diff --git a/openfoia/agent.py b/openfoia/agent.py
index 1657cd4..9c40531 100644
--- a/openfoia/agent.py
+++ b/openfoia/agent.py
@@ -12,12 +12,13 @@
from __future__ import annotations
+import contextlib
import logging
from dataclasses import dataclass
-from datetime import datetime
from typing import Any
from .models import Agency, Request, RequestStatus
+from .models import utcnow as _utcnow
logger = logging.getLogger(__name__)
@@ -318,10 +319,8 @@ async def _search_agencies(self, params: dict[str, Any]) -> dict[str, Any]:
if level and level != "all":
from .models import AgencyLevel
- try:
+ with contextlib.suppress(ValueError):
agencies = agencies.filter(Agency.level == AgencyLevel(level))
- except ValueError:
- pass
results = agencies.limit(20).all()
return {
@@ -385,6 +384,7 @@ async def _draft_request(self, params: dict[str, Any]) -> dict[str, Any]:
"""
import uuid
+
from .models import DeliveryMethod, User
request_id = str(uuid.uuid4())
@@ -399,9 +399,7 @@ async def _draft_request(self, params: dict[str, Any]) -> dict[str, Any]:
user = self.db.query(User).first()
if user and agency:
- from datetime import datetime as dt
-
- req_num = f"REQ-{dt.utcnow().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
+ req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid.uuid4().hex[:6].upper()}"
new_req = Request(
id=request_id,
request_number=req_num,
@@ -432,7 +430,7 @@ async def _send_request(self, params: dict[str, Any]) -> dict[str, Any]:
return {"error": f"Request not found: {request_id}"}
request.status = RequestStatus.SENT
- request.sent_at = datetime.utcnow()
+ request.sent_at = _utcnow()
# Auto-set due date (20 business days per FOIA statute)
if not request.due_date:
@@ -506,10 +504,8 @@ async def _list_requests(self, params: dict[str, Any]) -> dict[str, Any]:
status_filter = params.get("status")
if status_filter and status_filter != "all":
- try:
+ with contextlib.suppress(ValueError):
query = query.filter(Request.status == RequestStatus(status_filter))
- except ValueError:
- pass
agency_id = params.get("agency_id")
if agency_id:
@@ -541,6 +537,7 @@ async def _process_document(self, params: dict[str, Any]) -> dict[str, Any]:
database, where it becomes visible in reports and exports.
"""
from pathlib import Path
+
from .db import get_data_dir
doc_path = params.get("document_path", "")
@@ -604,7 +601,7 @@ async def _extract_entities(self, params: dict[str, Any]) -> dict[str, Any]:
async def _build_entity_graph(self, params: dict[str, Any]) -> dict[str, Any]:
"""Build entity graph."""
- from .models import Entity, Document, entity_links
+ from .models import Document, Entity, entity_links
query = self.db.query(Entity)
request_ids = params.get("request_ids")
@@ -639,10 +636,8 @@ async def _search_entities(self, params: dict[str, Any]) -> dict[str, Any]:
if entity_type and entity_type != "all":
from .models import EntityType
- try:
+ with contextlib.suppress(ValueError):
query = query.filter(Entity.entity_type == EntityType(entity_type))
- except ValueError:
- pass
results = query.limit(50).all()
diff --git a/openfoia/campaign.py b/openfoia/campaign.py
index 5c4c669..da7777c 100644
--- a/openfoia/campaign.py
+++ b/openfoia/campaign.py
@@ -23,6 +23,7 @@
RequestStatus,
User,
)
+from .models import utcnow as _utcnow
def _sandbox_env() -> SandboxedEnvironment:
@@ -71,7 +72,7 @@ def render(
context = {
"participant": participant,
"agency": agency,
- "date": datetime.utcnow().strftime("%B %d, %Y"),
+ "date": _utcnow().strftime("%B %d, %Y"),
"custom": custom_params or {},
}
@@ -167,7 +168,7 @@ async def generate_request(
)
# Generate request number
- request_number = f"REQ-{datetime.utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
+ request_number = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
# Determine delivery method
if agency.foia_email and template.recommended_method == DeliveryMethod.EMAIL:
@@ -211,7 +212,7 @@ async def schedule_staggered_send(
2. Overwhelming agency systems
3. Making it easy to identify and block
"""
- start = start_time or datetime.utcnow()
+ start = start_time or _utcnow()
schedule = []
# Distribute evenly with some randomness
@@ -307,7 +308,7 @@ async def generate_progress_report(self, campaign: Campaign) -> str:
- **Active:** {"Yes" if stats["is_active"] else "No"}
---
-*Generated: {datetime.utcnow().strftime("%Y-%m-%d %H:%M UTC")}*
+*Generated: {_utcnow().strftime("%Y-%m-%d %H:%M UTC")}*
""".strip()
diff --git a/openfoia/cli.py b/openfoia/cli.py
index 6a62f9f..c89978b 100644
--- a/openfoia/cli.py
+++ b/openfoia/cli.py
@@ -3,10 +3,11 @@
from __future__ import annotations
import asyncio
+import contextlib
import json
from datetime import datetime
from pathlib import Path
-from typing import Any, Optional
+from typing import Any
import typer
from rich import print as rprint
@@ -14,6 +15,8 @@
from rich.progress import Progress, SpinnerColumn, TextColumn
from rich.table import Table
+from .models import utcnow as _utcnow
+
app = typer.Typer(
name="openfoia",
help="Crowdsourced FOIA automation with AI-powered document analysis.",
@@ -36,7 +39,7 @@ def init(
duress: bool = typer.Option(
False, "--duress", help="Also set up a decoy database (prompts for a passphrase)"
),
- password: Optional[str] = typer.Option(
+ password: str | None = typer.Option(
None,
"--password",
prompt=False,
@@ -44,7 +47,7 @@ def init(
help="Encryption passphrase. Prefer --encrypt, which prompts: a passphrase "
"passed here is recorded in shell history and visible in the process list.",
),
- duress_password: Optional[str] = typer.Option(
+ duress_password: str | None = typer.Option(
None,
"--duress-password",
prompt=False,
@@ -71,7 +74,7 @@ def init(
openfoia init --encrypt # Initialize with encryption (prompts)
openfoia init --encrypt --duress # Also set up a decoy database
"""
- from .db import get_data_dir, get_db_path, init_db, has_sqlcipher
+ from .db import get_data_dir, get_db_path, has_sqlcipher, init_db
data_dir = get_data_dir()
@@ -113,7 +116,7 @@ def init(
# Show stats
from .db import get_session
- from .models import Agency, Request, Document
+ from .models import Agency, Document, Request
with get_session(password=password) as session:
agency_count = session.query(Agency).count()
@@ -382,7 +385,7 @@ def guide():
def serve(
port: int = typer.Option(0, "--port", "-p", help="Port to run on (0 = random)"),
host: str = typer.Option("127.0.0.1", "--host", "-h", help="Host to bind to"),
- browser: Optional[str] = typer.Option(
+ browser: str | None = typer.Option(
None, "--browser", "-b", help="Browser to open (safari/firefox/chrome/brave/tor)"
),
private: bool = typer.Option(
@@ -405,7 +408,7 @@ def serve(
import secrets
import socket
- from .browser import detect_browsers, launch_browser, print_browser_menu, BrowserType
+ from .browser import BrowserType, detect_browsers, launch_browser, print_browser_menu
# Generate session token for security
token = secrets.token_urlsafe(16)
@@ -473,8 +476,8 @@ def serve(
rprint("[yellow]No browser auto-selected. Copy the URL above.[/yellow]\n")
# Start the server
- from .server import run_server
from .db import get_data_dir
+ from .server import run_server
run_server(host=host, port=port, token=token, data_dir=get_data_dir())
@@ -556,7 +559,7 @@ def encrypt(
openfoia db encrypt --password SECRET
openfoia db encrypt # will prompt for password
"""
- from .db import get_db_path, encrypt_database, has_sqlcipher
+ from .db import encrypt_database, get_db_path, has_sqlcipher
if not has_sqlcipher():
rprint("[bold red]Error:[/bold red] pysqlcipher3 is not installed.")
@@ -575,7 +578,7 @@ def encrypt(
encrypt_database(password)
except Exception as e:
rprint(f"[bold red]Encryption failed:[/bold red] {e}")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
rprint("[bold green]Database encrypted successfully.[/bold green]")
rprint("[green]Plaintext database and its WAL/journal files were shredded in place.[/green]")
@@ -678,8 +681,8 @@ def config(
def request_new(
agency: str = typer.Option(..., "--agency", "-a", help="Target agency name or ID"),
subject: str = typer.Option(..., "--subject", "-s", help="Request subject"),
- body: Optional[str] = typer.Option(None, "--body", "-b", help="Request body (or use --file)"),
- body_file: Optional[Path] = typer.Option(
+ body: str | None = typer.Option(None, "--body", "-b", help="Request body (or use --file)"),
+ body_file: Path | None = typer.Option(
None, "--file", "-f", help="File containing request body"
),
method: str = typer.Option("email", "--method", "-m", help="Delivery method (email/fax/mail)"),
@@ -688,13 +691,18 @@ def request_new(
):
"""Create a new FOIA request."""
from uuid import uuid4
+
from .db import get_db_path, get_session, init_db
from .models import (
Agency as AgencyModel,
- Request as RequestModel,
- User,
- RequestStatus,
+ )
+ from .models import (
DeliveryMethod,
+ RequestStatus,
+ User,
+ )
+ from .models import (
+ Request as RequestModel,
)
if body_file:
@@ -735,7 +743,7 @@ def request_new(
session.flush()
# Create request
- req_num = f"REQ-{datetime.now().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
+ req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
try:
delivery = DeliveryMethod(method.lower())
@@ -776,13 +784,15 @@ def request_new(
@request_app.command("list")
def request_list(
- status: Optional[str] = typer.Option(None, "--status", "-s", help="Filter by status"),
- agency: Optional[str] = typer.Option(None, "--agency", "-a", help="Filter by agency"),
+ status: str | None = typer.Option(None, "--status", "-s", help="Filter by status"),
+ agency: str | None = typer.Option(None, "--agency", "-a", help="Filter by agency"),
limit: int = typer.Option(20, "--limit", "-n", help="Maximum results"),
):
"""List FOIA requests."""
- from .db import get_session, get_db_path
- from .models import Request as RequestModel, Agency as AgencyModel, RequestStatus
+ from .db import get_db_path, get_session
+ from .models import Agency as AgencyModel
+ from .models import Request as RequestModel
+ from .models import RequestStatus
db_path = get_db_path()
if not db_path.exists():
@@ -798,7 +808,7 @@ def request_list(
query = query.filter(RequestModel.status == status_enum)
except ValueError:
rprint(f"[red]Invalid status '{status}'.[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if agency:
query = query.filter(
@@ -849,8 +859,9 @@ def request_status(
request_id: str = typer.Argument(..., help="Request ID or number"),
):
"""Check status of a FOIA request."""
- from .db import get_session, get_db_path
- from .models import Request as RequestModel, TimelineEvent
+ from .db import get_db_path, get_session
+ from .models import Request as RequestModel
+ from .models import TimelineEvent
db_path = get_db_path()
if not db_path.exists():
@@ -930,17 +941,17 @@ def request_status(
def request_send(
agency: str = typer.Option(..., "--agency", "-a", help="Target agency (name or abbreviation)"),
subject: str = typer.Option(..., "--subject", "-s", help="Request subject"),
- body: Optional[str] = typer.Option(None, "--body", "-b", help="Request body text"),
- body_file: Optional[Path] = typer.Option(
+ body: str | None = typer.Option(None, "--body", "-b", help="Request body text"),
+ body_file: Path | None = typer.Option(
None, "--file", "-f", help="File containing request body"
),
- template: Optional[str] = typer.Option(
+ template: str | None = typer.Option(
None, "--template", "-t", help="Use template (standard/self)"
),
name: str = typer.Option(..., "--name", "-n", help="Your full name"),
email: str = typer.Option(..., "--email", "-e", help="Your email address"),
method: str = typer.Option("email", "--method", "-m", help="Delivery method (email/fax/mail)"),
- to_address: Optional[str] = typer.Option(
+ to_address: str | None = typer.Option(
None, "--to", help="Override recipient address (email, fax number, or mailing address)"
),
dry_run: bool = typer.Option(
@@ -971,9 +982,10 @@ def request_send(
openfoia request send -a FBI -s "Test" -t standard -n "Test User" -e test@example.com --dry-run
"""
import asyncio
+
from .db import get_db_path, get_session
- from .models import Agency
from .gateways.base import DeliveryPayload
+ from .models import Agency
if method not in ("email", "fax", "mail"):
rprint(f"[red]Unknown method '{method}'. Use: email, fax, mail[/red]")
@@ -1022,7 +1034,7 @@ def request_send(
# Get body content
if template:
- from .templates import standard_request, records_about_self, RequesterInfo, RequestDetails
+ from .templates import RequestDetails, RequesterInfo, records_about_self, standard_request
requester = RequesterInfo(name=name, email=email)
details = RequestDetails(subject=subject, description=subject)
@@ -1089,15 +1101,13 @@ def request_send(
from .db import get_data_dir
config_path = get_data_dir() / "config.json"
- import os
import json
+ import os
config = {}
if config_path.exists():
- try:
+ with contextlib.suppress(json.JSONDecodeError):
config = json.loads(config_path.read_text())
- except json.JSONDecodeError:
- pass
# Build and send via the appropriate gateway
if method == "email":
@@ -1206,10 +1216,13 @@ def request_send(
rprint(f" Expected delivery: {result.metadata['expected_delivery_date']}")
# Update the matching Request in the DB if one exists
- from .db import get_session, get_db_path
- from .models import Request as RequestModel, RequestStatus, Agency as AgencyModel
from datetime import timedelta
+ from .db import get_db_path, get_session
+ from .models import Agency as AgencyModel
+ from .models import Request as RequestModel
+ from .models import RequestStatus
+
db_path = get_db_path()
if db_path.exists():
with get_session() as session:
@@ -1227,7 +1240,7 @@ def request_send(
)
if req:
req.status = RequestStatus.SENT
- req.sent_at = datetime.now()
+ req.sent_at = _utcnow()
req.delivery_reference = result.reference_id
# Auto-set due date
response_days = req.agency.typical_response_days if req.agency else 20
@@ -1252,9 +1265,7 @@ def request_send(
@docs_app.command("ingest")
def docs_ingest(
path: Path = typer.Argument(..., help="File or directory to ingest"),
- request_id: Optional[str] = typer.Option(
- None, "--request", "-r", help="Associate with request"
- ),
+ request_id: str | None = typer.Option(None, "--request", "-r", help="Associate with request"),
recursive: bool = typer.Option(
True, "--recursive/--no-recursive", help="Recurse into directories"
),
@@ -1277,6 +1288,7 @@ def docs_ingest(
openfoia docs ingest ./doc.pdf --keep-metadata
"""
import asyncio
+
from .db import get_data_dir, get_db_path, init_db
from .pipeline.ingest import DocumentIngester
@@ -1359,9 +1371,10 @@ def docs_ingest(
# Persist Document rows to database
if results:
+ from uuid import uuid4
+
from .db import get_session
from .models import Document, DocumentType
- from uuid import uuid4
with get_session() as session:
for r in results:
@@ -1380,7 +1393,7 @@ def docs_ingest(
# Check if request_id is valid, otherwise create without it
if not request_id:
# Create a placeholder request for unassociated documents
- from .models import Request, User, RequestStatus, DeliveryMethod
+ from .models import DeliveryMethod, Request, RequestStatus, User
user = session.query(User).first()
if not user:
@@ -1448,7 +1461,7 @@ def docs_ocr(
backend: str = typer.Option(
"tesseract", "--backend", "-b", help="OCR backend (tesseract/google/aws)"
),
- output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output text file"),
+ output: Path | None = typer.Option(None, "--output", "-o", help="Output text file"),
):
"""Run OCR on a PDF document.
@@ -1464,6 +1477,7 @@ def docs_ocr(
openfoia docs ocr document.pdf --backend google
"""
import asyncio
+
from .pipeline.ocr import OCREngine, RedactionDetector
if not file_path.exists():
@@ -1489,10 +1503,10 @@ def docs_ocr(
rprint(f"[red]Missing dependency: {e}[/red]")
rprint("[dim]Install with: pip install pytesseract pdf2image[/dim]")
rprint("[dim]Also need: brew install tesseract poppler (macOS)[/dim]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
except Exception as e:
rprint(f"[red]OCR failed: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
progress.update(task, description="Detecting redactions...")
redactions = asyncio.run(detector.analyze(result.text, file_path))
@@ -1535,16 +1549,14 @@ def docs_ocr(
@agency_app.command("list")
def agency_list(
- level: Optional[str] = typer.Option(
+ level: str | None = typer.Option(
None, "--level", "-l", help="Filter by level (federal/state/local)"
),
- state: Optional[str] = typer.Option(
- None, "--state", "-s", help="Filter by state (2-letter code)"
- ),
+ state: str | None = typer.Option(None, "--state", "-s", help="Filter by state (2-letter code)"),
limit: int = typer.Option(50, "--limit", "-n", help="Maximum results"),
):
"""List agencies in the database."""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Agency, AgencyLevel
db_path = get_db_path()
@@ -1561,7 +1573,7 @@ def agency_list(
query = query.filter(Agency.level == level_enum)
except ValueError:
rprint(f"[red]Invalid level '{level}'. Use: federal, state, local, tribal[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if state:
query = query.filter(Agency.state == state.upper())
@@ -1598,7 +1610,7 @@ def agency_search(
limit: int = typer.Option(20, "--limit", "-n", help="Maximum results"),
):
"""Search for agencies by name or abbreviation."""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Agency
db_path = get_db_path()
@@ -1642,7 +1654,7 @@ def agency_info(
agency_id: str = typer.Argument(..., help="Agency abbreviation or name"),
):
"""Show detailed information about an agency."""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Agency
db_path = get_db_path()
@@ -1729,9 +1741,9 @@ def template_generate(
name: str = typer.Option(..., "--name", "-n", help="Your full name"),
email: str = typer.Option(..., "--email", "-e", help="Your email address"),
address: str = typer.Option("", "--address", help="Your mailing address"),
- organization: Optional[str] = typer.Option(None, "--org", help="Your organization"),
+ organization: str | None = typer.Option(None, "--org", help="Your organization"),
journalist: bool = typer.Option(False, "--journalist", "-j", help="You are a journalist"),
- output: Optional[Path] = typer.Option(
+ output: Path | None = typer.Option(
None, "--output", "-o", help="Output file (default: stdout)"
),
no_fee_waiver: bool = typer.Option(
@@ -1745,7 +1757,7 @@ def template_generate(
openfoia template generate standard -a FBI -s "Records on X" -n "Jane Doe" -e jane@example.com
openfoia template generate standard -a EPA -s "Pollution data" -n "John Smith" -e john@example.com -j
"""
- from .templates import standard_request, records_about_self, RequesterInfo, RequestDetails
+ from .templates import RequestDetails, RequesterInfo, records_about_self, standard_request
# Build requester info
requester = RequesterInfo(
@@ -1889,6 +1901,7 @@ def campaign_create(
):
"""Create a new crowdsourced campaign."""
from uuid import uuid4
+
from .db import get_db_path, get_session, init_db
from .models import Campaign, User
@@ -1932,7 +1945,7 @@ def campaign_create(
@campaign_app.command("list")
def campaign_list():
"""List all campaigns."""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Campaign
db_path = get_db_path()
@@ -1974,7 +1987,7 @@ def campaign_status(
campaign_id: str = typer.Argument(..., help="Campaign ID (or prefix)"),
):
"""Check campaign progress."""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Campaign, RequestStatus
db_path = get_db_path()
@@ -2035,7 +2048,8 @@ def campaign_join(
):
"""Join a campaign as a participant."""
from uuid import uuid4
- from .db import get_session, get_db_path
+
+ from .db import get_db_path, get_session
from .models import Campaign, User
db_path = get_db_path()
@@ -2091,13 +2105,18 @@ def campaign_distribute(
target agency and assigns them round-robin to participants.
"""
from uuid import uuid4
- from .db import get_session, get_db_path
+
+ from .db import get_db_path, get_session
from .models import (
- Campaign,
Agency as AgencyModel,
- Request as RequestModel,
- RequestStatus,
+ )
+ from .models import (
+ Campaign,
DeliveryMethod,
+ RequestStatus,
+ )
+ from .models import (
+ Request as RequestModel,
)
db_path = get_db_path()
@@ -2163,7 +2182,7 @@ def campaign_distribute(
skipped += 1
continue
- req_num = f"REQ-{datetime.now().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
+ req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
request = RequestModel(
id=str(uuid4()),
request_number=req_num,
@@ -2195,8 +2214,9 @@ def campaign_progress(
campaign_id: str = typer.Argument(..., help="Campaign ID (or prefix)"),
):
"""Show per-participant, per-agency status grid for a campaign."""
- from .db import get_session, get_db_path
- from .models import Campaign, Agency as AgencyModel
+ from .db import get_db_path, get_session
+ from .models import Agency as AgencyModel
+ from .models import Campaign
db_path = get_db_path()
if not db_path.exists():
@@ -2291,9 +2311,9 @@ def campaign_progress(
@analyze_app.command("extract")
def analyze_extract(
document_id: str = typer.Argument(..., help="Document ID to analyze"),
- output: Optional[Path] = typer.Option(None, "--output", "-o", help="Output file"),
+ output: Path | None = typer.Option(None, "--output", "-o", help="Output file"),
force: bool = typer.Option(False, "--force", help="Re-extract even if already done"),
- model: Optional[str] = typer.Option(
+ model: str | None = typer.Option(
None, "--model", "-m", help="LLM model (e.g. llama3.1:8b, llama3.2:3b)"
),
ensemble: bool = typer.Option(
@@ -2305,7 +2325,7 @@ def analyze_extract(
Pipeline: regex + NER → merge → LLM validation (if available).
Use --ensemble to run ALL NER backends (GLiNER + spaCy) together.
"""
- from .db import get_session, get_db_path
+ from .db import get_db_path, get_session
from .models import Document, Entity
db_path = get_db_path()
@@ -2355,6 +2375,7 @@ def analyze_extract(
# Run extraction
import asyncio
+
from .pipeline.extract import EntityExtractor
extractor = EntityExtractor(model=model) if model else EntityExtractor()
@@ -2377,7 +2398,7 @@ def analyze_extract(
except Exception as e:
rprint(f"[red]Extraction failed: {e}[/red]")
rprint("[dim]Ensure AI provider is configured: openfoia config --init[/dim]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if not result.entities:
rprint("[yellow]No entities found in document.[/yellow]")
@@ -2385,6 +2406,7 @@ def analyze_extract(
# Save entities to database
from uuid import uuid4
+
from .models import entity_links
entity_id_map: dict[str, str] = {} # normalized_text.lower() -> entity.id
@@ -2521,7 +2543,8 @@ def analyze_graphs_list():
size = f.stat().st_size
size_str = f"{size / 1024:.0f}KB" if size > 1024 else f"{size}B"
- modified = datetime.fromtimestamp(f.stat().st_mtime).strftime("%Y-%m-%d %H:%M")
+ # Local time is intended: this is a file listing shown to the user.
+ modified = datetime.fromtimestamp(f.stat().st_mtime).strftime("%Y-%m-%d %H:%M") # noqa: DTZ006
table.add_row(name, " + ".join(types), size_str, modified)
@@ -2532,13 +2555,11 @@ def analyze_graphs_list():
@analyze_app.command("graph")
def analyze_graph(
- request_id: Optional[str] = typer.Option(
- None, "--request", "-r", help="Analyze single request"
- ),
- campaign_id: Optional[str] = typer.Option(
+ request_id: str | None = typer.Option(None, "--request", "-r", help="Analyze single request"),
+ campaign_id: str | None = typer.Option(
None, "--campaign", "-c", help="Analyze entire campaign"
),
- name: Optional[str] = typer.Option(
+ name: str | None = typer.Option(
None, "--name", "-n", help="Save as named graph (stored in ~/.openfoia/graphs/)"
),
output: Path = typer.Option(
@@ -2559,8 +2580,9 @@ def analyze_graph(
openfoia analyze graph --request REQ-001 --name epa # filter + save
openfoia analyze graphs # list saved graphs
"""
- from .db import get_session, get_db_path
- from .models import Entity, Document, Request as RequestModel, entity_links
+ from .db import get_db_path, get_session
+ from .models import Document, Entity, entity_links
+ from .models import Request as RequestModel
db_path = get_db_path()
if not db_path.exists():
@@ -2627,9 +2649,11 @@ def analyze_graph(
doc_ids = {e.document_id for e in entities if e.document_id}
documents = {}
if doc_ids:
- from .models import Document as DocModel, Request as ReqModel
import re as re_mod
+ from .models import Document as DocModel
+ from .models import Request as ReqModel
+
for doc in session.query(DocModel).filter(DocModel.id.in_(doc_ids)).all():
# Derive source URL from request body or filename
source_url = None
@@ -2710,10 +2734,7 @@ def _load_config_data() -> tuple[Path, dict]:
from .db import get_data_dir
config_path = get_data_dir() / "config.json"
- if config_path.exists():
- data = json.loads(config_path.read_text())
- else:
- data = {}
+ data = json.loads(config_path.read_text()) if config_path.exists() else {}
return config_path, data
@@ -2766,7 +2787,7 @@ def entities_add(
re.compile(pattern)
except re.error as e:
rprint(f"[red]Invalid regex pattern: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
name = name.upper().replace(" ", "_")
@@ -2858,8 +2879,8 @@ def _fuzzy_match_column(header: str) -> str | None:
def _llm_map_columns(headers: list[str], sample_rows: list[list[str]]) -> dict[str, int] | None:
"""Use the configured LLM to figure out which columns map to name/pattern/description."""
- from .pipeline.extract import _llm_available, _call_ollama
from .config import load_config
+ from .pipeline.extract import _call_ollama, _llm_available
cfg = load_config()
if not _llm_available(cfg.ai.provider, cfg.ai.api_key, cfg.ai.base_url):
@@ -2913,8 +2934,8 @@ def _llm_generate_regex(description: str) -> str | None:
2. Matches at least one example from the description (if examples are present)
3. Is reasonably short (not hallucinated garbage)
"""
- from .pipeline.extract import _llm_available
from .config import load_config
+ from .pipeline.extract import _llm_available
cfg = load_config()
if not _llm_available(cfg.ai.provider, cfg.ai.api_key, cfg.ai.base_url):
@@ -3218,7 +3239,7 @@ def entities_export(
@entities_app.command("test")
def entities_test(
text: str = typer.Option(None, "--text", "-t", help="Test text (or reads from stdin)"),
- file: Optional[Path] = typer.Option(None, "--file", "-f", help="Test against a file"),
+ file: Path | None = typer.Option(None, "--file", "-f", help="Test against a file"),
):
"""Test your custom entity types against sample text.
@@ -3303,8 +3324,10 @@ def deadline_list(
Federal agencies have 20 business days to respond (5 U.S.C. 552).
This command shows what's due, what's overdue, and what needs follow-up.
"""
- from .db import get_session, get_db_path
- from .models import Request as RequestModel, Agency as AgencyModel, RequestStatus
+ from .db import get_db_path, get_session
+ from .models import Agency as AgencyModel
+ from .models import Request as RequestModel
+ from .models import RequestStatus
db_path = get_db_path()
if not db_path.exists():
@@ -3362,7 +3385,7 @@ def deadline_list(
table.add_column("Days Over", style="red")
for r in overdue:
- days_over = (datetime.utcnow() - r.due_date).days
+ days_over = (_utcnow() - r.due_date).days
table.add_row(
r.request_number,
r.agency.abbreviation or r.agency.name,
@@ -3384,7 +3407,7 @@ def deadline_list(
table.add_column("Days Left", style="green")
for r in upcoming:
- days_left = (r.due_date - datetime.utcnow()).days
+ days_left = (r.due_date - _utcnow()).days
color = "green" if days_left > 5 else "yellow"
table.add_row(
r.request_number,
@@ -3411,8 +3434,9 @@ def deadline_check():
Example (add to .bashrc):
openfoia deadlines check 2>/dev/null
"""
- from .db import get_session, get_db_path
- from .models import Request as RequestModel, RequestStatus
+ from .db import get_db_path, get_session
+ from .models import Request as RequestModel
+ from .models import RequestStatus
db_path = get_db_path()
if not db_path.exists():
@@ -3440,7 +3464,7 @@ def deadline_check():
r.due_date = _foia_due_date(r.sent_at)
if r.is_overdue():
overdue_count += 1
- days_over = (datetime.utcnow() - r.due_date).days
+ days_over = (_utcnow() - r.due_date).days
rprint(f"[red]OVERDUE:[/red] {r.request_number} — {r.subject} (+{days_over} days)")
if overdue_count:
@@ -3481,6 +3505,7 @@ def browse(
openfoia browse https://example.com --tor --headless --save # Headless Tor
"""
import asyncio
+
from .tor_browse import browse as _browse
try:
@@ -3493,10 +3518,10 @@ def browse(
)
)
except SystemExit:
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
except Exception as e:
rprint(f"[red]Browse failed:[/red] {e}")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
rprint(f"\n[cyan]Title:[/cyan] {result.get('title', 'N/A')}")
rprint(f"[cyan]URL:[/cyan] {result.get('url', url)}")
@@ -3542,6 +3567,7 @@ def purge(
print_ssd_warning,
secure_delete_dir,
)
+
from .db import get_data_dir as _get_data_dir
data_dir = _get_data_dir()
@@ -3619,9 +3645,7 @@ def purge(
def ingest_url(
url: str = typer.Option(..., "--url", "-u", help="URL to fetch and ingest"),
tor: bool = typer.Option(False, "--tor", help="Route through Tor SOCKS5 proxy"),
- output: Optional[Path] = typer.Option(
- None, "--output", "-o", help="Save extracted text to file"
- ),
+ output: Path | None = typer.Option(None, "--output", "-o", help="Save extracted text to file"),
):
"""Ingest a web page into the document pipeline.
@@ -3636,6 +3660,7 @@ def ingest_url(
openfoia ingest --url https://example.onion/docs --tor
"""
import asyncio
+
from .db import get_data_dir
from .pipeline.web import archive_url
@@ -3655,7 +3680,7 @@ def ingest_url(
rprint(f"[red]Failed to fetch URL: {e}[/red]")
if tor:
rprint("[dim]Make sure Tor is running: brew install tor && tor[/dim]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
rprint("\n[bold green]Archived web page[/bold green]")
rprint("=" * 50)
@@ -3696,10 +3721,10 @@ def records_search(
"-s",
help="Data source (muckrock, opencorporates, sec)",
),
- jurisdiction: Optional[str] = typer.Option(
+ jurisdiction: str | None = typer.Option(
None, "--jurisdiction", "-j", help="Jurisdiction filter (e.g. us_ca, gb)"
),
- filing_type: Optional[str] = typer.Option(
+ filing_type: str | None = typer.Option(
None, "--type", "-t", help="Filing type filter for SEC (e.g. 10-K, 8-K)"
),
limit: int = typer.Option(10, "--limit", "-n", help="Maximum results to display"),
@@ -3722,6 +3747,7 @@ def records_search(
openfoia records search "EPA water" --source muckrock
"""
import asyncio
+
from .records import get_adapter, list_sources
# Validate source
@@ -3749,7 +3775,7 @@ def records_search(
result = asyncio.run(adapter.search(query, **kwargs))
except Exception as e:
rprint(f"[red]Search failed: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if raw:
rprint(
@@ -3995,7 +4021,7 @@ def records_fetch(
result_id, text = asyncio.run(adapter.pull_text(doc_id))
except Exception as e:
rprint(f"[red]Fetch failed: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if not result_id or not text:
rprint(f"[red]Could not fetch text for document {doc_id}.[/red]")
@@ -4051,7 +4077,7 @@ def records_download(
entity = asyncio.run(adapter.fetch(request_id))
except Exception as e:
rprint(f"[red]Failed to fetch request: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
if not entity:
rprint(f"[red]Request {request_id} not found on MuckRock.[/red]")
@@ -4084,7 +4110,7 @@ def records_download(
downloaded = asyncio.run(adapter.download_files(request_id, str(output)))
except Exception as e:
rprint(f"[red]Download failed: {e}[/red]")
- raise typer.Exit(1)
+ raise typer.Exit(1) from None
rprint(f"\n[green]{len(downloaded)} file(s) downloaded to {output}/[/green]")
@@ -4119,10 +4145,11 @@ def records_download(
# Persist Document rows to database
if ingest_results:
- from .db import get_session
- from .models import Document, DocumentType, Request, User, RequestStatus, DeliveryMethod
from uuid import uuid4
+ from .db import get_session
+ from .models import DeliveryMethod, Document, DocumentType, Request, RequestStatus, User
+
with get_session() as session:
# Create a placeholder request for downloaded docs
user = session.query(User).first()
@@ -4175,22 +4202,22 @@ def records_download(
@app.command()
def crossref(
- request_id: Optional[str] = typer.Option(
+ request_id: str | None = typer.Option(
None, "--request", "-r", help="Cross-ref entities from a specific request"
),
- document_id: Optional[str] = typer.Option(
+ document_id: str | None = typer.Option(
None, "--document", "-d", help="Cross-ref entities from a specific document"
),
- sources: Optional[str] = typer.Option(
+ sources: str | None = typer.Option(
None,
"--sources",
help="Comma-separated sources (muckrock,opencorporates,sec,opensanctions,documentcloud)",
),
- icij_data: Optional[Path] = typer.Option(
+ icij_data: Path | None = typer.Option(
None, "--icij-data", help="Path to downloaded ICIJ CSV data"
),
- output: Optional[Path] = typer.Option(None, "--output", "-o", help="Save report to file"),
- ftm: Optional[Path] = typer.Option(
+ output: Path | None = typer.Option(None, "--output", "-o", help="Save report to file"),
+ ftm: Path | None = typer.Option(
None, "--ftm", help="Export results as FollowTheMoney JSON-lines"
),
yes: bool = typer.Option(False, "--yes", "-y", help="Skip the network confirmation prompt"),
@@ -4209,9 +4236,10 @@ def crossref(
openfoia crossref --icij-data ./icij-csvs/ # include Offshore Leaks
openfoia crossref --ftm results.ftm.json # export as FollowTheMoney
"""
- from .db import get_session, get_db_path
- from .models import Entity, Document, Request as RequestModel
from .crossref import crossref_entities
+ from .db import get_db_path, get_session
+ from .models import Document, Entity
+ from .models import Request as RequestModel
db_path = get_db_path()
if not db_path.exists():
@@ -4299,9 +4327,7 @@ def crossref(
rprint("[bold]Cross-referencing entities...[/bold]")
def _progress(event: str, msg: str) -> None:
- if event == "start":
- rprint(f"[dim] {msg}[/dim]")
- elif event == "entity":
+ if event == "start" or event == "entity":
rprint(f"[dim] {msg}[/dim]")
report = asyncio.run(
@@ -4393,7 +4419,7 @@ def _progress(event: str, msg: str) -> None:
@analyze_app.command("export")
def analyze_export(
output: Path = typer.Option("entities.ftm.json", "--output", "-o", help="Output file path"),
- request_id: Optional[str] = typer.Option(
+ request_id: str | None = typer.Option(
None, "--request", "-r", help="Export from specific request"
),
):
@@ -4406,10 +4432,11 @@ def analyze_export(
openfoia analyze export
openfoia analyze export -o investigation.ftm.json -r REQ-20260322-ABC
"""
- from .db import get_session, get_db_path
- from .models import Entity, Document, Request as RequestModel, entity_links
- from .pipeline.extract import ExtractedEntity
+ from .db import get_db_path, get_session
from .ftm import export_ftm
+ from .models import Document, Entity, entity_links
+ from .models import Request as RequestModel
+ from .pipeline.extract import ExtractedEntity
db_path = get_db_path()
if not db_path.exists():
@@ -4465,7 +4492,7 @@ def analyze_export(
@analyze_app.command("import")
def analyze_import(
file: Path = typer.Argument(..., help="FtM JSON-lines file to import"),
- tag: Optional[str] = typer.Option(None, "--tag", "-t", help="Tag for this import batch"),
+ tag: str | None = typer.Option(None, "--tag", "-t", help="Tag for this import batch"),
):
"""Import entities from a FollowTheMoney JSON-lines file.
diff --git a/openfoia/config.py b/openfoia/config.py
index e2dc191..9771fb6 100644
--- a/openfoia/config.py
+++ b/openfoia/config.py
@@ -13,6 +13,7 @@
from __future__ import annotations
+import contextlib
import json
import os
from dataclasses import dataclass, field
@@ -425,7 +426,6 @@ def save_config(config: OpenFOIAConfig, config_path: Path | str | None = None) -
json.dump(data, f, indent=2)
if os.name != "nt":
- try:
- os.chmod(path, 0o600) # tighten a pre-existing looser file
- except OSError:
- pass
+ # Tighten a pre-existing looser file; best-effort on odd filesystems.
+ with contextlib.suppress(OSError):
+ os.chmod(path, 0o600)
diff --git a/openfoia/crossref.py b/openfoia/crossref.py
index 38be96b..a808330 100644
--- a/openfoia/crossref.py
+++ b/openfoia/crossref.py
@@ -89,8 +89,6 @@ class _RateLimited(BaseException):
in individual checkers don't swallow it — the crossref loop catches it.
"""
- pass
-
def _check_rate_limit(result: Any) -> None:
"""Raise _RateLimited if the search result indicates a rate limit error."""
diff --git a/openfoia/db.py b/openfoia/db.py
index 60b39c1..12140b3 100644
--- a/openfoia/db.py
+++ b/openfoia/db.py
@@ -12,9 +12,10 @@
import sqlite3
import stat
import tempfile
-from contextlib import contextmanager
+from collections.abc import Generator
+from contextlib import contextmanager, suppress
from pathlib import Path
-from typing import Any, Generator
+from typing import Any
from sqlalchemy import create_engine, event
from sqlalchemy.engine import Engine
@@ -195,7 +196,7 @@ def get_db_path(password: str | None = None) -> Path:
password = get_db_password()
if password:
- from .security import is_duress_password, get_decoy_db_path
+ from .security import get_decoy_db_path, is_duress_password
if is_duress_password(password):
return get_decoy_db_path()
@@ -335,10 +336,8 @@ def _restrict_db_permissions(db_path: Path) -> None:
return
for candidate in (db_path, *(Path(str(db_path) + s) for s in _DB_SIDECAR_SUFFIXES)):
if candidate.is_file():
- try:
+ with suppress(OSError):
os.chmod(candidate, 0o600)
- except OSError:
- pass
def encrypt_database(password: str) -> None:
diff --git a/openfoia/ftm.py b/openfoia/ftm.py
index 18c1ed1..6fb66d0 100644
--- a/openfoia/ftm.py
+++ b/openfoia/ftm.py
@@ -10,12 +10,11 @@
from __future__ import annotations
import json
-from typing import Any
from pathlib import Path
+from typing import Any
from .models import EntityType
-
# Map OpenFOIA entity types to FtM schema types
_ENTITY_TYPE_TO_FTM_SCHEMA: dict[str, str] = {
"person": "Person",
@@ -67,7 +66,7 @@
def _try_ftm_available() -> bool:
"""Check if followthemoney library is installed."""
try:
- import followthemoney # noqa: F401
+ import followthemoney # noqa: F401 - availability probe for the optional extra
return True
except ImportError:
diff --git a/openfoia/ftm_import.py b/openfoia/ftm_import.py
index fd36b87..0181c66 100644
--- a/openfoia/ftm_import.py
+++ b/openfoia/ftm_import.py
@@ -192,7 +192,7 @@ def import_ftm_to_db(
Returns:
(entities_imported, relationships_imported)
"""
- from .db import get_session, get_db_path, init_db
+ from .db import get_db_path, get_session, init_db
from .models import Entity
db_path = get_db_path()
@@ -293,7 +293,7 @@ def _get_or_create_import_doc(session: Any, tag: str | None = None) -> str:
if _IMPORT_DOC_ID:
return _IMPORT_DOC_ID
- from .models import Document, DocumentType, Request, User, Agency, RequestStatus, DeliveryMethod
+ from .models import Agency, DeliveryMethod, Document, DocumentType, Request, RequestStatus, User
# Need a request to hang the document on
user = session.query(User).first()
diff --git a/openfoia/gateways/__init__.py b/openfoia/gateways/__init__.py
index a1d45f8..57e72e5 100644
--- a/openfoia/gateways/__init__.py
+++ b/openfoia/gateways/__init__.py
@@ -8,14 +8,14 @@
"""
from .base import DeliveryGateway, DeliveryResult
+from .email import EmailGateway
from .fax import TwilioFaxGateway
from .mail import LobMailGateway
-from .email import EmailGateway
__all__ = [
"DeliveryGateway",
"DeliveryResult",
- "TwilioFaxGateway",
- "LobMailGateway",
"EmailGateway",
+ "LobMailGateway",
+ "TwilioFaxGateway",
]
diff --git a/openfoia/gateways/email.py b/openfoia/gateways/email.py
index b1b8a36..f998ad6 100644
--- a/openfoia/gateways/email.py
+++ b/openfoia/gateways/email.py
@@ -5,12 +5,12 @@
import asyncio
import smtplib
import ssl
-from datetime import datetime
from email.mime.application import MIMEApplication
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.utils import getaddresses
+from ..models import utcnow as _utcnow
from .base import DeliveryGateway, DeliveryPayload, DeliveryResult, DeliveryStatus
@@ -138,13 +138,13 @@ def _send():
import hashlib
ref_id = hashlib.sha256(
- f"{payload.recipient_address}:{payload.subject}:{datetime.utcnow().isoformat()}".encode()
+ f"{payload.recipient_address}:{payload.subject}:{_utcnow().isoformat()}".encode()
).hexdigest()[:16]
return DeliveryResult(
status=DeliveryStatus.SENT,
reference_id=ref_id,
- sent_at=datetime.utcnow(),
+ sent_at=_utcnow(),
cost_cents=0, # Email is free (sort of)
metadata={
"to": payload.recipient_address,
@@ -164,16 +164,17 @@ def _send():
async def _send_sendgrid(self, payload: DeliveryPayload) -> DeliveryResult:
"""Send via SendGrid API."""
try:
+ import base64
+
from sendgrid import SendGridAPIClient
from sendgrid.helpers.mail import (
- Mail,
Attachment,
+ Disposition,
FileContent,
FileName,
FileType,
- Disposition,
+ Mail,
)
- import base64
message = Mail(
from_email=(self.from_email, self.from_name),
@@ -206,7 +207,7 @@ async def _send_sendgrid(self, payload: DeliveryPayload) -> DeliveryResult:
return DeliveryResult(
status=DeliveryStatus.SENT,
reference_id=message_id,
- sent_at=datetime.utcnow(),
+ sent_at=_utcnow(),
cost_cents=0,
metadata={
"to": payload.recipient_address,
@@ -256,7 +257,7 @@ def _format_email_body(self, payload: DeliveryPayload) -> str:
---
REQUEST DETAILS
Subject: {payload.subject}
-Date: {datetime.utcnow().strftime("%B %d, %Y")}
+Date: {_utcnow().strftime("%B %d, %Y")}
I request a fee waiver for this request. Disclosure of the requested information is in the public interest because it is likely to contribute significantly to public understanding of government operations and activities.
diff --git a/openfoia/gateways/fax.py b/openfoia/gateways/fax.py
index 82267bb..9a14897 100644
--- a/openfoia/gateways/fax.py
+++ b/openfoia/gateways/fax.py
@@ -16,7 +16,7 @@
import io
import logging
import os
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
@@ -138,7 +138,7 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult:
return DeliveryResult(
status=DeliveryStatus.PENDING,
reference_id=fax.sid,
- sent_at=datetime.now(timezone.utc),
+ sent_at=datetime.now(UTC),
cost_cents=pages * self.COST_PER_PAGE_CENTS,
metadata={
"to": payload.recipient_address,
@@ -268,9 +268,9 @@ def _generate_fax_pdf(self, payload: DeliveryPayload) -> bytes:
def _generate_pdf_reportlab(self, payload: DeliveryPayload) -> bytes:
"""Generate PDF using reportlab with proper legal formatting."""
from reportlab.lib.pagesizes import letter
- from reportlab.lib.styles import getSampleStyleSheet, ParagraphStyle
+ from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
from reportlab.lib.units import inch
- from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer
+ from reportlab.platypus import Paragraph, SimpleDocTemplate, Spacer
buffer = io.BytesIO()
doc = SimpleDocTemplate(
@@ -311,9 +311,7 @@ def _generate_pdf_reportlab(self, payload: DeliveryPayload) -> bytes:
story.append(Paragraph(f"TO: {_esc(payload.recipient_name)}", body_style))
story.append(Paragraph(f"FAX: {_esc(payload.recipient_address)}", body_style))
story.append(
- Paragraph(
- f"DATE: {datetime.now(timezone.utc).strftime('%B %d, %Y')}", body_style
- )
+ Paragraph(f"DATE: {datetime.now(UTC).strftime('%B %d, %Y')}", body_style)
)
story.append(
Paragraph(f"RE: FOIA Request - {_esc(payload.subject)}", body_style)
@@ -392,7 +390,7 @@ def _generate_pdf_minimal(self, payload: DeliveryPayload) -> bytes:
This is a fallback that creates a bare-bones but valid PDF.
"""
- date_str = datetime.now(timezone.utc).strftime("%B %d, %Y")
+ date_str = datetime.now(UTC).strftime("%B %d, %Y")
sender = (payload.return_address or "[Requester Name]").split("\n")[0]
text_lines = [
diff --git a/openfoia/gateways/mail.py b/openfoia/gateways/mail.py
index 1ab98f1..81bfd47 100644
--- a/openfoia/gateways/mail.py
+++ b/openfoia/gateways/mail.py
@@ -14,7 +14,7 @@
import html
import logging
import re
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from typing import Any
from .base import DeliveryGateway, DeliveryPayload, DeliveryResult, DeliveryStatus
@@ -94,7 +94,8 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult:
sends it via Lob's print-and-mail API, and returns tracking info.
"""
try:
- import lob # noqa: F401 — triggers ImportError if not installed
+ # Availability probe: raises ImportError if the extra is missing.
+ import lob # noqa: F401
lob_client = self._get_lob()
@@ -142,7 +143,7 @@ async def send(self, payload: DeliveryPayload) -> DeliveryResult:
return DeliveryResult(
status=DeliveryStatus.SENT,
reference_id=letter.id,
- sent_at=datetime.now(timezone.utc),
+ sent_at=datetime.now(UTC),
cost_cents=self.estimate_cost(payload),
metadata={
"tracking_number": tracking_number,
@@ -274,7 +275,7 @@ def _estimate_pages(self, payload: DeliveryPayload) -> int:
pages = max(1, len(payload.body) // 3000 + 1)
if payload.attachments:
- for filename, content in payload.attachments:
+ for _filename, content in payload.attachments:
pages += max(1, len(content) // 3000 + 1)
return pages
@@ -353,7 +354,7 @@ def _generate_letter_html(self, payload: DeliveryPayload) -> str:
Lob renders HTML to PDF for printing. The template uses standard
fonts and margins suitable for USPS mailing.
"""
- date_str = datetime.now(timezone.utc).strftime("%B %d, %Y")
+ date_str = datetime.now(UTC).strftime("%B %d, %Y")
sender_name = self.return_address.get("name", "[Requester Name]")
# Build sender address block for letterhead
diff --git a/openfoia/graph_template.py b/openfoia/graph_template.py
index 684bd3c..3c25a1c 100644
--- a/openfoia/graph_template.py
+++ b/openfoia/graph_template.py
@@ -32,8 +32,8 @@ def escape_json_for_script(graph_json: str) -> str:
graph_json.replace("&", "\\u0026")
.replace("<", "\\u003c")
.replace(">", "\\u003e")
- .replace("
", "\\u2028")
- .replace("
", "\\u2029")
+ .replace("\u2028", "\\u2028")
+ .replace("\u2029", "\\u2029")
)
diff --git a/openfoia/migrations/versions/001_initial_schema.py b/openfoia/migrations/versions/001_initial_schema.py
index 2f40e6b..0b8931f 100644
--- a/openfoia/migrations/versions/001_initial_schema.py
+++ b/openfoia/migrations/versions/001_initial_schema.py
@@ -6,17 +6,16 @@
"""
-from typing import Sequence, Union
+from collections.abc import Sequence
-from alembic import op
import sqlalchemy as sa
-
+from alembic import op
# revision identifiers, used by Alembic.
revision: str = "001"
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
diff --git a/openfoia/models.py b/openfoia/models.py
index 9a9d2d9..d7c3539 100644
--- a/openfoia/models.py
+++ b/openfoia/models.py
@@ -11,7 +11,7 @@
from __future__ import annotations
import enum
-from datetime import datetime
+from datetime import UTC, datetime
from typing import Any
from uuid import uuid4
@@ -24,13 +24,30 @@
ForeignKey,
Integer,
String,
- Text,
Table,
+ Text,
create_engine,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+def utcnow() -> datetime:
+ """Current UTC time as a **naive** datetime.
+
+ The ORM's ``DateTime`` columns are timezone-naive, and model helpers such
+ as ``Request.days_pending`` compare stored values against "now". Returning
+ an aware datetime here would raise
+ ``TypeError: can't subtract offset-naive and offset-aware datetimes``
+ against every existing row.
+
+ This replaces the deprecated ``utcnow()`` (removed in a future
+ Python; CI already targets 3.13) while preserving the naive-UTC storage
+ convention exactly. Moving to timezone-aware columns is a separate,
+ deliberate migration.
+ """
+ return datetime.now(UTC).replace(tzinfo=None)
+
+
class Base(DeclarativeBase):
pass
@@ -135,11 +152,11 @@ class User(Base):
name: Mapped[str] = mapped_column(String(255))
organization: Mapped[str | None] = mapped_column(String(255), nullable=True)
is_journalist: Mapped[bool] = mapped_column(default=False)
- created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+ created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
# Relationships
- requests: Mapped[list["Request"]] = relationship(back_populates="requester")
- campaigns: Mapped[list["Campaign"]] = relationship(
+ requests: Mapped[list[Request]] = relationship(back_populates="requester")
+ campaigns: Mapped[list[Campaign]] = relationship(
secondary=campaign_participants, back_populates="participants"
)
@@ -174,7 +191,7 @@ class Agency(Base):
total_requests_tracked: Mapped[int] = mapped_column(Integer, default=0)
# Relationships
- requests: Mapped[list["Request"]] = relationship(back_populates="agency")
+ requests: Mapped[list[Request]] = relationship(back_populates="agency")
class Request(Base):
@@ -209,7 +226,7 @@ class Request(Base):
agency_tracking_number: Mapped[str | None] = mapped_column(String(100), nullable=True)
# Dates
- created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+ created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
sent_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
acknowledged_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
due_date: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
@@ -224,23 +241,23 @@ class Request(Base):
extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
# Relationships
- requester: Mapped["User"] = relationship(back_populates="requests")
- agency: Mapped["Agency"] = relationship(back_populates="requests")
- campaign: Mapped["Campaign | None"] = relationship(back_populates="requests")
- documents: Mapped[list["Document"]] = relationship(back_populates="request")
- timeline: Mapped[list["TimelineEvent"]] = relationship(back_populates="request")
+ requester: Mapped[User] = relationship(back_populates="requests")
+ agency: Mapped[Agency] = relationship(back_populates="requests")
+ campaign: Mapped[Campaign | None] = relationship(back_populates="requests")
+ documents: Mapped[list[Document]] = relationship(back_populates="request")
+ timeline: Mapped[list[TimelineEvent]] = relationship(back_populates="request")
def days_pending(self) -> int:
"""Days since request was sent."""
if not self.sent_at:
return 0
- return (datetime.utcnow() - self.sent_at).days
+ return (utcnow() - self.sent_at).days
def is_overdue(self) -> bool:
"""Whether the request is past its due date."""
if not self.due_date:
return False
- return datetime.utcnow() > self.due_date
+ return utcnow() > self.due_date
class Document(Base):
@@ -271,12 +288,12 @@ class Document(Base):
) # e.g., ["b(6)", "b(7)(A)"]
# Dates
- received_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+ received_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
processed_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# Relationships
- request: Mapped["Request"] = relationship(back_populates="documents")
- entities: Mapped[list["Entity"]] = relationship(back_populates="source_document")
+ request: Mapped[Request] = relationship(back_populates="documents")
+ entities: Mapped[list[Entity]] = relationship(back_populates="source_document")
class Entity(Base):
@@ -304,8 +321,8 @@ class Entity(Base):
extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
# Relationships
- source_document: Mapped["Document"] = relationship(back_populates="entities")
- linked_entities: Mapped[list["Entity"]] = relationship(
+ source_document: Mapped[Document] = relationship(back_populates="entities")
+ linked_entities: Mapped[list[Entity]] = relationship(
secondary=entity_links,
primaryjoin=id == entity_links.c.source_id,
secondaryjoin=id == entity_links.c.target_id,
@@ -333,14 +350,14 @@ class Campaign(Base):
# Status
is_active: Mapped[bool] = mapped_column(default=True)
- created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+ created_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
ends_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True)
# Relationships
- participants: Mapped[list["User"]] = relationship(
+ participants: Mapped[list[User]] = relationship(
secondary=campaign_participants, back_populates="campaigns"
)
- requests: Mapped[list["Request"]] = relationship(back_populates="campaign")
+ requests: Mapped[list[Request]] = relationship(back_populates="campaign")
def request_count(self) -> int:
return len(self.requests)
@@ -364,11 +381,11 @@ class TimelineEvent(Base):
String(50)
) # sent, acknowledged, response, appeal, etc.
description: Mapped[str] = mapped_column(Text)
- occurred_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow)
+ occurred_at: Mapped[datetime] = mapped_column(DateTime, default=utcnow)
extra_data: Mapped[dict[str, Any] | None] = mapped_column("metadata", JSON, nullable=True)
# Relationships
- request: Mapped["Request"] = relationship(back_populates="timeline")
+ request: Mapped[Request] = relationship(back_populates="timeline")
# === Database Setup ===
diff --git a/openfoia/pipeline/__init__.py b/openfoia/pipeline/__init__.py
index d2196c9..b76078f 100644
--- a/openfoia/pipeline/__init__.py
+++ b/openfoia/pipeline/__init__.py
@@ -8,12 +8,12 @@
5. Link - Connect entities across documents
"""
+from .extract import EntityExtractor
from .ingest import DocumentIngester
from .ocr import OCREngine
-from .extract import EntityExtractor
__all__ = [
"DocumentIngester",
- "OCREngine",
"EntityExtractor",
+ "OCREngine",
]
diff --git a/openfoia/pipeline/extract.py b/openfoia/pipeline/extract.py
index d260c05..18b1eff 100644
--- a/openfoia/pipeline/extract.py
+++ b/openfoia/pipeline/extract.py
@@ -16,9 +16,8 @@
from dataclasses import dataclass, field
from typing import Any
-from ..config import load_config, OpenFOIAConfig, EntityConfig
-from ..models import EntityType, ConfidenceLevel
-
+from ..config import EntityConfig, OpenFOIAConfig, load_config
+from ..models import ConfidenceLevel, EntityType
# ---------------------------------------------------------------------------
# Data classes
@@ -211,7 +210,7 @@ def _is_protected_acronym(raw: str) -> bool:
return s in _KEEP_ACRONYMS or (s.isupper() and 2 <= len(s) <= 6 and s.isalpha())
-def _mention_score(m: "Mention") -> float:
+def _mention_score(m: Mention) -> float:
"""Score a mention for quality. 0.0 = definitely junk, should be dropped."""
s = _surface_clean(m.normalized_text)
k = s.lower().rstrip(".")
@@ -272,7 +271,7 @@ def _ocr_fold(text: str) -> str:
return re.sub(r"[^a-z0-9 ]+", "", t)
-def _core_tokens(text: str, etype: "EntityType") -> list[str]:
+def _core_tokens(text: str, etype: EntityType) -> list[str]:
"""Extract meaningful tokens, stripping org suffixes."""
toks = re.findall(r"[a-z0-9]+", _ocr_fold(text))
if etype == EntityType.ORGANIZATION:
@@ -415,7 +414,7 @@ def _get_gliner():
def _gliner_available() -> bool:
"""Check if GLiNER is installed."""
try:
- import gliner # noqa: F401
+ import gliner # noqa: F401 - availability probe for the optional extra
return True
except ImportError:
@@ -1836,10 +1835,11 @@ def _group_key(m: Mention) -> tuple[str, str]:
a, b = c1.canonical_text, c2.canonical_text
# Substring match
- if a.lower() in b.lower() or b.lower() in a.lower():
- should_merge = True
- # OCR-fold similarity
- elif SequenceMatcher(None, _ocr_fold(a), _ocr_fold(b)).ratio() >= 0.90:
+ if (
+ a.lower() in b.lower()
+ or b.lower() in a.lower()
+ or SequenceMatcher(None, _ocr_fold(a), _ocr_fold(b)).ratio() >= 0.90
+ ):
should_merge = True
# Token Jaccard for orgs
elif c1.entity_type == EntityType.ORGANIZATION:
@@ -2021,10 +2021,11 @@ def _find_or_create_canonical(self, entity: ExtractedEntity) -> str:
return can_id
can_norm = canonical["normalized"].lower()
- if normalized in can_norm or can_norm in normalized:
- if len(normalized) > 3 and len(can_norm) > 3:
- canonical["aliases"].add(entity.raw_text)
- return can_id
+ if (normalized in can_norm or can_norm in normalized) and (
+ len(normalized) > 3 and len(can_norm) > 3
+ ):
+ canonical["aliases"].add(entity.raw_text)
+ return can_id
import uuid
diff --git a/openfoia/pipeline/ingest.py b/openfoia/pipeline/ingest.py
index 31e847d..6ebfc66 100644
--- a/openfoia/pipeline/ingest.py
+++ b/openfoia/pipeline/ingest.py
@@ -7,12 +7,12 @@
import mimetypes
import shutil
from dataclasses import dataclass
-from datetime import datetime
from pathlib import Path
from typing import Any
from uuid import uuid4
from ..models import DocumentType
+from ..models import utcnow as _utcnow
@dataclass
@@ -103,7 +103,7 @@ async def ingest_file(
checksum=checksum,
metadata={
"original_path": str(file_path),
- "ingested_at": datetime.utcnow().isoformat(),
+ "ingested_at": _utcnow().isoformat(),
"doc_type": doc_type.value,
"request_id": request_id,
"metadata_stripped": stripped_info if stripped_info else None,
@@ -159,7 +159,7 @@ async def ingest_bytes(
extracted_text=extracted_text,
checksum=checksum,
metadata={
- "ingested_at": datetime.utcnow().isoformat(),
+ "ingested_at": _utcnow().isoformat(),
"doc_type": doc_type.value,
"request_id": request_id,
**(metadata or {}),
diff --git a/openfoia/pipeline/metadata.py b/openfoia/pipeline/metadata.py
index c24e4a6..e99a216 100644
--- a/openfoia/pipeline/metadata.py
+++ b/openfoia/pipeline/metadata.py
@@ -252,9 +252,7 @@ def _strip_docx_metadata(file_path: Path, keep_hash: bool) -> dict[str, Any]:
# Clear sensitive properties
for attr in _DOCX_SENSITIVE_ATTRS:
try:
- if attr in ("created", "modified"):
- setattr(core, attr, None)
- elif attr == "revision":
+ if attr in ("created", "modified") or attr == "revision":
setattr(core, attr, None)
else:
setattr(core, attr, "")
diff --git a/openfoia/pipeline/ocr.py b/openfoia/pipeline/ocr.py
index 91d7e83..6b35d4a 100644
--- a/openfoia/pipeline/ocr.py
+++ b/openfoia/pipeline/ocr.py
@@ -6,7 +6,7 @@
import tempfile
from dataclasses import dataclass
from pathlib import Path
-from typing import Any
+from typing import Any, ClassVar
#: Hard cap on pages rasterized in one OCR run. A hostile "FOIA response"
#: declaring tens of thousands of pages would otherwise exhaust memory/disk
@@ -147,7 +147,11 @@ def _convert():
for i, image in enumerate(images):
# Get detailed OCR data
- def _ocr():
+ # `image` is bound as a default so the closure captures THIS
+ # iteration's page. It is awaited immediately today, so late
+ # binding does not bite — but parallelizing this loop later would
+ # otherwise silently OCR the last page N times.
+ def _ocr(image=image):
data = pytesseract.image_to_data(image, output_type=pytesseract.Output.DICT)
text = pytesseract.image_to_string(image)
return data, text
@@ -311,7 +315,7 @@ class RedactionDetector:
"""Detect and analyze redactions in documents."""
# Common FOIA exemption patterns
- EXEMPTION_PATTERNS = {
+ EXEMPTION_PATTERNS: ClassVar[dict] = {
r"\(b\)\(1\)": "National security",
r"\(b\)\(2\)": "Internal personnel rules",
r"\(b\)\(3\)": "Statutory exemption",
@@ -358,8 +362,8 @@ async def _count_visual_redactions(self, pdf_path: Path) -> int:
Uses image analysis to detect large black rectangular regions.
"""
- from pdf2image import convert_from_path
import numpy as np
+ from pdf2image import convert_from_path
def _analyze():
with tempfile.TemporaryDirectory(dir=get_ocr_temp_dir()) as scratch:
diff --git a/openfoia/pipeline/web.py b/openfoia/pipeline/web.py
index 54a09cb..3fefe7f 100644
--- a/openfoia/pipeline/web.py
+++ b/openfoia/pipeline/web.py
@@ -6,16 +6,16 @@
from __future__ import annotations
+import contextlib
import hashlib
import re
from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from uuid import uuid4
-
# Known tracker/analytics domains and script patterns to strip
TRACKER_PATTERNS: list[str] = [
r"google-analytics\.com",
@@ -120,10 +120,9 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None
self._in_article += 1
elif tag == "main":
self._in_main += 1
- elif tag == "div":
- if self._current_div_text is None:
- self._current_div_text = []
- self._div_depth = len(self._tag_stack)
+ elif tag == "div" and self._current_div_text is None:
+ self._current_div_text = []
+ self._div_depth = len(self._tag_stack)
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
@@ -141,10 +140,13 @@ def handle_endtag(self, tag: str) -> None:
self._in_article -= 1
elif tag == "main" and self._in_main > 0:
self._in_main -= 1
- elif tag == "div" and self._current_div_text is not None:
- if len(self._tag_stack) <= self._div_depth:
- self._div_blocks.append(self._current_div_text)
- self._current_div_text = None
+ elif (
+ tag == "div"
+ and self._current_div_text is not None
+ and len(self._tag_stack) <= self._div_depth
+ ):
+ self._div_blocks.append(self._current_div_text)
+ self._current_div_text = None
if self._tag_stack and self._tag_stack[-1] == tag:
self._tag_stack.pop()
@@ -253,10 +255,9 @@ def _extract_content(html: str) -> tuple[str, str]:
Returns (text_content, page_title).
"""
extractor = _ContentExtractor()
- try:
+ # Best-effort parsing: malformed markup must not abort the archive.
+ with contextlib.suppress(Exception):
extractor.feed(html)
- except Exception:
- pass # Best-effort parsing
return extractor.get_content(), extractor.title.strip()
@@ -293,7 +294,7 @@ async def fetch_url(url: str, use_tor: bool = False) -> WebFetchResult:
# Extract main content
text, title = _extract_content(cleaned_html)
- fetched_at = datetime.now(timezone.utc).isoformat()
+ fetched_at = datetime.now(UTC).isoformat()
return WebFetchResult(
url=url,
diff --git a/openfoia/records/__init__.py b/openfoia/records/__init__.py
index 0cc57c9..8ac3f02 100644
--- a/openfoia/records/__init__.py
+++ b/openfoia/records/__init__.py
@@ -39,15 +39,15 @@ def list_sources() -> list[str]:
def _auto_register() -> None:
"""Auto-register built-in adapters."""
- from .opencorporates import OpenCorporatesAdapter
- from .sec_edgar import SECEdgarAdapter
- from .muckrock import MuckRockAdapter
from .documentcloud import DocumentCloudAdapter
- from .usaspending import USASpendingAdapter
- from .propublica_nonprofit import ProPublicaNonprofitAdapter
- from .govinfo import GovInfoAdapter
from .fec import FECAdapter
+ from .govinfo import GovInfoAdapter
+ from .muckrock import MuckRockAdapter
+ from .opencorporates import OpenCorporatesAdapter
+ from .propublica_nonprofit import ProPublicaNonprofitAdapter
from .regulations import RegulationsGovAdapter
+ from .sec_edgar import SECEdgarAdapter
+ from .usaspending import USASpendingAdapter
register("opencorporates", OpenCorporatesAdapter)
register("sec", SECEdgarAdapter)
diff --git a/openfoia/records/muckrock.py b/openfoia/records/muckrock.py
index c56a41f..a82f699 100644
--- a/openfoia/records/muckrock.py
+++ b/openfoia/records/muckrock.py
@@ -11,7 +11,7 @@
from __future__ import annotations
import logging
-from typing import Any
+from typing import Any, ClassVar
import httpx
@@ -179,7 +179,7 @@ async def search(
)
# Common abbreviation → full agency name mapping
- _AGENCY_NAMES: dict[str, str] = {
+ _AGENCY_NAMES: ClassVar[dict[str, str]] = {
"fbi": "Federal Bureau of Investigation",
"cia": "Central Intelligence Agency",
"nsa": "National Security Agency",
diff --git a/openfoia/records/sec_edgar.py b/openfoia/records/sec_edgar.py
index 1e20bf8..c7ab652 100644
--- a/openfoia/records/sec_edgar.py
+++ b/openfoia/records/sec_edgar.py
@@ -8,7 +8,6 @@
from typing import Any
-
from .base import AdapterRequestError, RecordAdapter, RecordEntity, SearchResult
EFTS_BASE = "https://efts.sec.gov/LATEST/search-index"
diff --git a/openfoia/security.py b/openfoia/security.py
index fc96940..39d37d8 100644
--- a/openfoia/security.py
+++ b/openfoia/security.py
@@ -13,6 +13,7 @@
from __future__ import annotations
+import contextlib
import os
import shutil
import tempfile
@@ -20,6 +21,7 @@
from rich import print as rprint
+from .models import utcnow as _utcnow
# ---------------------------------------------------------------------------
# Secure file deletion
@@ -76,16 +78,13 @@ def secure_delete_dir(path: Path | str) -> int:
secure_delete(item)
count += 1
elif item.is_dir():
- try:
+ # Non-empty dir — will retry after its children are removed.
+ with contextlib.suppress(OSError):
item.rmdir()
- except OSError:
- pass # non-empty dir — will retry after children removed
# Remove the root directory itself
- try:
+ with contextlib.suppress(OSError):
path.rmdir()
- except OSError:
- pass
return count
@@ -171,16 +170,12 @@ def fill_free_space(path: Path | str, chunk_size_mb: int = 100) -> None:
# Clean up fill files
for fp in fill_files:
- try:
+ with contextlib.suppress(OSError):
fp.unlink()
- except OSError:
- pass
# Try to remove the directory if we created it
- try:
+ with contextlib.suppress(OSError):
path.rmdir()
- except OSError:
- pass
# ---------------------------------------------------------------------------
@@ -399,7 +394,7 @@ def seed_decoy_db(db_path: Path, password: str | None = None) -> None:
If password is provided, the decoy is encrypted with SQLCipher.
"""
import random
- from datetime import datetime, timedelta
+ from datetime import timedelta
from uuid import uuid4
from sqlalchemy import create_engine
@@ -510,7 +505,7 @@ def _creator():
session.flush()
# --- Requests (bland, non-sensitive) ---
- now = datetime.utcnow()
+ now = _utcnow()
requests_data = [
{
"subject": "Monthly weather data summaries for Portland, OR (2024)",
diff --git a/openfoia/server.py b/openfoia/server.py
index b55eb3d..6ad888f 100644
--- a/openfoia/server.py
+++ b/openfoia/server.py
@@ -6,19 +6,20 @@
from __future__ import annotations
+import contextlib
import secrets
-from datetime import datetime
from pathlib import Path
from uuid import uuid4
-from fastapi import FastAPI, Request, HTTPException, Depends, Query, UploadFile, File
-from fastapi.responses import HTMLResponse
+from fastapi import Depends, FastAPI, File, HTTPException, Query, Request, UploadFile
from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import HTMLResponse
from fastapi.staticfiles import StaticFiles
-from starlette.middleware.trustedhost import TrustedHostMiddleware
from pydantic import BaseModel
from sqlalchemy import func
+from starlette.middleware.trustedhost import TrustedHostMiddleware
+from .models import utcnow as _utcnow
#: Hard cap on a single uploaded document (100 MiB), matching CLI ingest.
MAX_UPLOAD_BYTES = 100 * 1024 * 1024
@@ -138,11 +139,13 @@ async def stats(token: str = Depends(verify_token)):
"""Get overview statistics."""
from .db import get_session
from .models import (
- Request as FOIARequest,
Document,
Entity,
- RequestStatus,
EntityType,
+ RequestStatus,
+ )
+ from .models import (
+ Request as FOIARequest,
)
with get_session() as session:
@@ -225,7 +228,8 @@ async def list_requests(
):
"""List FOIA requests."""
from .db import get_session
- from .models import Request as FOIARequest, RequestStatus, Agency
+ from .models import Agency, RequestStatus
+ from .models import Request as FOIARequest
with get_session() as session:
query = session.query(FOIARequest).join(Agency)
@@ -238,7 +242,7 @@ async def list_requests(
raise HTTPException(
status_code=400,
detail=f"Invalid status '{status}'. Valid: {', '.join(s.value for s in RequestStatus)}",
- )
+ ) from None
requests = query.order_by(FOIARequest.created_at.desc()).limit(limit).all()
@@ -267,11 +271,13 @@ async def create_request(
"""Create a new FOIA request."""
from .db import get_session
from .models import (
- Request as FOIARequest,
Agency,
- User,
- RequestStatus,
DeliveryMethod,
+ RequestStatus,
+ User,
+ )
+ from .models import (
+ Request as FOIARequest,
)
with get_session() as session:
@@ -290,7 +296,7 @@ async def create_request(
session.add(user)
session.flush()
- req_num = f"REQ-{datetime.utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
+ req_num = f"REQ-{_utcnow().strftime('%Y%m%d')}-{uuid4().hex[:6].upper()}"
try:
method = DeliveryMethod(request_data.method)
@@ -426,7 +432,7 @@ async def upload_document(
raise HTTPException(
status_code=413,
detail=f"File exceeds the {MAX_UPLOAD_BYTES // (1024 * 1024)} MiB upload limit.",
- )
+ ) from None
except Exception:
dest.unlink(missing_ok=True)
raise
@@ -439,10 +445,8 @@ async def upload_document(
ingester = DocumentIngester(storage_path=docs_dir)
extracted_text = None
- try:
+ with contextlib.suppress(Exception):
extracted_text = await ingester._extract_text(dest, mime)
- except Exception:
- pass
from .db import get_session
from .models import Document, DocumentType
@@ -514,7 +518,8 @@ async def list_entities(
async def deadlines(token: str = Depends(verify_token)):
"""Get upcoming deadlines for pending requests."""
from .db import get_session
- from .models import Request as FOIARequest, RequestStatus, Agency
+ from .models import Agency, RequestStatus
+ from .models import Request as FOIARequest
with get_session() as session:
active_statuses = [
@@ -560,7 +565,7 @@ async def get_graph(
):
"""Get entity relationship graph."""
from .db import get_session
- from .models import Entity, Document, entity_links
+ from .models import Document, Entity, entity_links
with get_session() as session:
q = session.query(Entity)
@@ -1244,6 +1249,7 @@ def run_server(
) -> None:
"""Run the OpenFOIA server."""
import socket
+
import uvicorn
# Generate token if not provided
diff --git a/openfoia/templates.py b/openfoia/templates.py
index 111771b..f7c1043 100644
--- a/openfoia/templates.py
+++ b/openfoia/templates.py
@@ -12,7 +12,6 @@
from dataclasses import dataclass
from datetime import datetime
-from typing import Optional
@dataclass
@@ -20,13 +19,13 @@ class RequesterInfo:
"""Information about the person filing the request."""
name: str
- organization: Optional[str] = None
+ organization: str | None = None
address: str = ""
email: str = ""
phone: str = ""
is_journalist: bool = False
is_educational: bool = False
- publication: Optional[str] = None
+ publication: str | None = None
@dataclass
@@ -35,10 +34,10 @@ class RequestDetails:
subject: str
description: str
- date_range_start: Optional[datetime] = None
- date_range_end: Optional[datetime] = None
- keywords: list[str] = None
- exclusions: Optional[str] = None
+ date_range_start: datetime | None = None
+ date_range_end: datetime | None = None
+ keywords: list[str] | None = None
+ exclusions: str | None = None
# === Base Request Template ===
@@ -57,7 +56,9 @@ def standard_request(
This is the core template that works for most federal agencies.
Uses language proven to be effective based on RCFP guidance.
"""
- date_str = datetime.now().strftime("%B %d, %Y")
+ # Local date is intended: this is the date printed on a letter the
+ # requester is sending, not a stored timestamp.
+ date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005
# Build date range clause if provided
date_clause = ""
@@ -243,7 +244,9 @@ def appeal_denial(
Appeals must generally be filed within 90 days of the denial.
"""
- date_str = datetime.now().strftime("%B %d, %Y")
+ # Local date is intended: this is the date printed on a letter the
+ # requester is sending, not a stored timestamp.
+ date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005
request_date = original_request_date.strftime("%B %d, %Y")
denial_date_str = denial_date.strftime("%B %d, %Y")
@@ -348,7 +351,9 @@ def records_about_self(
This template combines FOIA and Privacy Act requests for maximum coverage.
"""
- date_str = datetime.now().strftime("%B %d, %Y")
+ # Local date is intended: this is the date printed on a letter the
+ # requester is sending, not a stored timestamp.
+ date_str = datetime.now().strftime("%B %d, %Y") # noqa: DTZ005
letter = f"""{date_str}
@@ -412,7 +417,7 @@ def records_about_self(
# === CLI Integration ===
-def list_templates() -> list[dict]:
+def list_templates() -> list[dict[str, str]]:
"""Return list of available templates for CLI display."""
return [
{
diff --git a/openfoia/tor_browse.py b/openfoia/tor_browse.py
index 1fe4f9d..794bbe4 100644
--- a/openfoia/tor_browse.py
+++ b/openfoia/tor_browse.py
@@ -13,11 +13,13 @@
from __future__ import annotations
+import contextlib
from pathlib import Path
from typing import Any
from rich import print as rprint
+from .models import utcnow as _utcnow
# ---------------------------------------------------------------------------
# Constants
@@ -82,7 +84,7 @@ async def browse(
"[yellow]Run: openfoia install-extras browser[/yellow]\n"
"[dim]Then: playwright install chromium[/dim]"
)
- raise SystemExit(1)
+ raise SystemExit(1) from None
if use_tor:
rprint(_TOR_WARNING)
@@ -182,17 +184,16 @@ async def browse(
# Sanitize filename from URL
import re
- from datetime import datetime
safe_name = re.sub(r"[^\w\-.]", "_", url.split("//", 1)[-1][:60])
- timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
+ timestamp = _utcnow().strftime("%Y%m%d_%H%M%S")
filename = f"{timestamp}_{safe_name}.txt"
out_path = save_dir / filename
out_path.write_text(
f"URL: {result['url']}\n"
f"Title: {result['title']}\n"
- f"Captured: {datetime.utcnow().isoformat()}Z\n"
+ f"Captured: {_utcnow().isoformat()}Z\n"
f"Tor: {use_tor}\n"
f"{'=' * 60}\n\n"
f"{content}"
@@ -204,10 +205,8 @@ async def browse(
rprint(
"\n[dim]Browser is open. Close the browser window or press Ctrl+C to exit.[/dim]"
)
- try:
+ with contextlib.suppress(KeyboardInterrupt):
await page.wait_for_event("close", timeout=0)
- except KeyboardInterrupt:
- pass
await browser.close()
diff --git a/pyproject.toml b/pyproject.toml
index 4ccc489..bc9052f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -76,21 +76,51 @@ line-length = 100
target-version = "py311"
[tool.ruff.lint]
-# Pin the rule set explicitly. CI installs an unpinned `ruff>=0.1.0`, so
-# relying on ruff's *defaults* means a new ruff release can silently widen
-# the rule set and turn CI red on unchanged code — which is what happened.
+# The rule set is pinned explicitly, never inherited from ruff's defaults:
+# CI installs an unpinned `ruff>=0.1.0`, so relying on defaults lets a new
+# ruff release silently change what CI enforces (which is exactly how this
+# repo went from green to 325 errors on unchanged code).
#
-# This list is a FLOOR, not a target. It reproduces the rules the project was
-# already passing, so lint is deterministic again; it is not a judgement that
-# the wider rules are worthless. Several of them have real signal for this
-# codebase — notably DTZ (naive/`utcnow()` datetimes, deprecated on the 3.13
-# CI target) and BLE001/S110 (blind excepts, the exact shape of the bug where
-# a failed API lookup was reported as a clean result).
-#
-# Widening this set and fixing the ~300 resulting findings is tracked as
-# follow-up work; it is a repo-wide diff that does not belong in a security
-# change. Widen deliberately here — never by upgrading ruff.
-select = ["E4", "E7", "E9", "F"]
+# Widen this list deliberately, and fix the findings — do not narrow it to
+# make a build pass.
+select = [
+ "E4", "E7", "E9", # pycodestyle (the historical default subset)
+ "F", # pyflakes
+ "I", # import sorting
+ "UP", # pyupgrade
+ "B", # bugbear
+ "SIM", # simplify
+ "DTZ", # naive datetimes — caught a real deadline bug
+ "C4", # comprehensions
+ "PIE",
+ "RUF",
+]
+
+ignore = [
+ # `class X(str, Enum)` -> `StrEnum` changes what `str(X.MEMBER)` returns
+ # ("federal" instead of "AgencyLevel.FEDERAL"). These enums are persisted
+ # through SQLAlchemy and serialized into API responses and exports, so
+ # that is a data-format change wearing a lint fix's clothing. Worth doing
+ # deliberately with migration checks, not as part of a lint sweep.
+ "UP042",
+]
+
+# Not enabled yet, tracked as follow-up:
+# BLE001 (~82) — blind `except Exception`. High signal for this codebase:
+# a swallowed exception is what made a failed records lookup report as a
+# clean result. Fixing it needs a per-site audit of error handling, which
+# is its own reviewable change rather than a bulk sweep.
+# S (bandit) — mostly subprocess/randomness false positives here, plus
+# S101 which is just "asserts in tests".
+
+[tool.ruff.lint.per-file-ignores]
+# FastAPI's dependency-injection idiom is `arg = Depends(...)` in the
+# signature default; B008 flags every route.
+"openfoia/server.py" = ["B008"]
+# Typer's CLI idiom is the same: `arg: T = typer.Option(...)`.
+"openfoia/cli.py" = ["B008"]
+# Tests legitimately use `assert`, private access, and broad `raises`.
+"tests/*" = ["S101", "SLF001"]
[tool.mypy]
python_version = "3.11"
diff --git a/tests/benchmark_extraction.py b/tests/benchmark_extraction.py
index 21c0fe6..2ce7846 100644
--- a/tests/benchmark_extraction.py
+++ b/tests/benchmark_extraction.py
@@ -15,12 +15,12 @@
import time
from pathlib import Path
+from openfoia.config import load_config
from openfoia.pipeline.extract import (
EntityExtractor,
_gliner_available,
_llm_available,
)
-from openfoia.config import load_config
# Richer test doc with more entities and relationships
DOC = """
@@ -312,7 +312,7 @@ def main():
p_found, p_total, p_missed = check_recall(result, EXPECTED_PERSONS, "person")
o_found, o_total, o_missed = check_recall(result, EXPECTED_ORGS, "organization")
- m_found, m_total, m_missed = check_recall(result, EXPECTED_MONEY, "money")
+ m_found, m_total, _m_missed = check_recall(result, EXPECTED_MONEY, "money")
r_found, r_total, r_missed = check_relationship_recall(result, EXPECTED_RELATIONSHIPS)
pr = p_found / p_total if p_total else 0
diff --git a/tests/test_datetime_semantics.py b/tests/test_datetime_semantics.py
new file mode 100644
index 0000000..b263a00
--- /dev/null
+++ b/tests/test_datetime_semantics.py
@@ -0,0 +1,112 @@
+"""Regression tests for the naive-UTC datetime convention.
+
+`datetime.utcnow()` is deprecated (3.12+) and this project's CI targets 3.13,
+so it has to go. But the ORM columns are `DateTime` **without** timezone, i.e.
+naive UTC, and model helpers compare stored values against "now". Swapping in
+an aware `datetime.now(timezone.utc)` would make those comparisons raise
+`TypeError: can't subtract offset-naive and offset-aware datetimes`.
+
+These tests pin the convention: keep storing naive UTC, but stop calling the
+deprecated API. Changing to fully timezone-aware storage is a separate,
+deliberate migration.
+"""
+
+from __future__ import annotations
+
+import warnings
+from datetime import UTC, datetime, timedelta
+
+
+def test_utcnow_helper_returns_naive_datetime():
+ """Storage stays naive so it stays comparable with existing rows."""
+ from openfoia.models import utcnow
+
+ now = utcnow()
+
+ assert now.tzinfo is None, "helper returned an aware datetime; ORM columns are naive"
+
+
+def test_utcnow_helper_is_actually_utc():
+ from openfoia.models import utcnow
+
+ delta = abs((utcnow() - datetime.now(UTC).replace(tzinfo=None)).total_seconds())
+
+ assert delta < 5, "helper is not UTC-based"
+
+
+def test_utcnow_helper_emits_no_deprecation_warning():
+ """The whole point: no deprecated utcnow() under the hood."""
+ from openfoia.models import utcnow
+
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", DeprecationWarning)
+ utcnow()
+
+
+def test_days_pending_still_works_against_naive_storage():
+ """This is what a blind aware-datetime conversion would break."""
+ from openfoia.models import Request, utcnow
+
+ req = Request()
+ req.sent_at = utcnow() - timedelta(days=3)
+
+ assert req.days_pending() == 3
+
+
+def test_is_overdue_still_works_against_naive_storage():
+ from openfoia.models import Request, utcnow
+
+ req = Request()
+ req.due_date = utcnow() - timedelta(days=1)
+ assert req.is_overdue() is True
+
+ req.due_date = utcnow() + timedelta(days=1)
+ assert req.is_overdue() is False
+
+
+def test_no_local_time_written_into_utc_columns():
+ """`sent_at = datetime.now()` wrote LOCAL time into a UTC column.
+
+ Every reader (`days_pending`, `is_overdue`) compares those values against
+ UTC, so for a user in UTC+9 or UTC-8 the statutory FOIA deadline was off
+ by up to a day. Timestamps destined for storage must use the helper.
+ """
+ import re
+ from pathlib import Path
+
+ import openfoia
+
+ pkg = Path(openfoia.__file__).parent
+ offenders = []
+ # Assignments of a bare datetime.now() to a persisted timestamp column.
+ pattern = re.compile(
+ r"\.(sent_at|created_at|received_at|processed_at|acknowledged_at|"
+ r"completed_at|occurred_at|due_date|ends_at)\s*=\s*datetime\.now\(\s*\)"
+ )
+ for path in pkg.rglob("*.py"):
+ for lineno, line in enumerate(path.read_text().splitlines(), 1):
+ if pattern.search(line):
+ offenders.append(f"{path.relative_to(pkg)}:{lineno}")
+
+ assert offenders == [], f"local time written into a naive-UTC column: {offenders}"
+
+
+def test_no_deprecated_utcnow_calls_remain_in_package():
+ """`datetime.utcnow()` is deprecated and removed-in-future."""
+ import re
+ from pathlib import Path
+
+ import openfoia
+
+ pkg = Path(openfoia.__file__).parent
+ offenders = []
+ for path in pkg.rglob("*.py"):
+ for lineno, line in enumerate(path.read_text().splitlines(), 1):
+ if line.strip().startswith("#"):
+ continue
+ # Attribute-style calls only: `datetime.utcnow()` and aliased
+ # forms like `dt.utcnow()`. A bare `utcnow()` is our own helper.
+ if re.search(r"\w\.utcnow\s*\(", line):
+ offenders.append(f"{path.relative_to(pkg)}:{lineno}")
+
+ assert offenders == [], f"deprecated datetime.utcnow() still called: {offenders}"
diff --git a/tests/test_extraction.py b/tests/test_extraction.py
index 0adf11c..7546e10 100644
--- a/tests/test_extraction.py
+++ b/tests/test_extraction.py
@@ -6,16 +6,17 @@
"""
import asyncio
+
import pytest
+
from openfoia.pipeline.extract import (
EntityExtractor,
ExtractionResult,
_gliner_available,
- _spacy_available,
_llm_available,
+ _spacy_available,
)
-
# ---------------------------------------------------------------------------
# Test documents with ground truth
# ---------------------------------------------------------------------------
diff --git a/tests/test_security_gateways.py b/tests/test_security_gateways.py
index 5f37ae3..02742d5 100644
--- a/tests/test_security_gateways.py
+++ b/tests/test_security_gateways.py
@@ -9,6 +9,7 @@
import asyncio
import os
+import re
import stat
import pytest
@@ -252,7 +253,7 @@ def stream(self, method, url):
return _Stream(_Resp())
dest = tmp_path / "out.bin"
- with pytest.raises(ValueError, match="(?i)size|large|bytes"):
+ with pytest.raises(ValueError, match=re.compile("size|large|bytes", re.I)):
asyncio.run(download_to_file(_Client(), "https://x.test/a.pdf", dest, max_bytes=8 * 1024))
# It must have stopped early, not consumed the whole stream.
diff --git a/tests/test_security_injection.py b/tests/test_security_injection.py
index 45a7ae6..a4dd225 100644
--- a/tests/test_security_injection.py
+++ b/tests/test_security_injection.py
@@ -67,10 +67,10 @@ def test_graph_render_escapes_line_separators(tmp_path):
"""U+2028/U+2029 are valid JSON but break JS string literals."""
html = _render_graph(
tmp_path,
- {"nodes": [], "links": [], "documents": [{"id": "d1", "text": "a
b
c"}]},
+ {"nodes": [], "links": [], "documents": [{"id": "d1", "text": "a\u2028b\u2029c"}]},
)
- assert "
" not in html
- assert "
" not in html
+ assert "\u2028" not in html
+ assert "\u2029" not in html
def test_graph_data_still_round_trips(tmp_path):