diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..b8e2d32 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,16 @@ +{ + "name": "openfoia", + "owner": { + "name": "Jordan Coin Jackson" + }, + "metadata": { + "description": "Claude Code plugin for OpenFOIA — local-first FOIA automation and AI document analysis for journalists and researchers." + }, + "plugins": [ + { + "name": "openfoia", + "source": "./plugin", + "description": "Help journalists and researchers run FOIA investigations using the openfoia CLI. Ships a skill covering the full records-search → download → OCR → extract → crossref → graph loop plus slash commands for each phase." + } + ] +} diff --git a/.gitignore b/.gitignore index dc622ba..09ea6fc 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,6 @@ Thumbs.db # Logs *.log logs/ + +# Generated benchmark artifact (tests/benchmark_extraction.py) +test_graph.html diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b6f4fc9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,191 @@ +# Changelog + +All notable changes to OpenFOIA are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Entries before 4.0.0 are backfilled from git history and are summaries, not +exhaustive lists. + +## [4.1.0] - 2026-08-11 + +Additive release on top of the 4.0.0 security hardening. No security +regressions: the 4.0.0 network egress choke point, graph escaping, CDN +removal, and fail-closed installer verification are all preserved. + +### Added + +- A Claude Code plugin (`plugin/`) — an `openfoia` skill plus four slash + commands (`/foia-install`, `/foia-search`, `/foia-investigate`, + `/foia-graph`). Install with + `/plugin marketplace add JordanCoin/openfoia` then + `/plugin install openfoia@openfoia`. Every documented `openfoia ...` + invocation is validated against the live 4.0.0 command tree, including the + new `egress-status` command. +- `openfoia --version` (also `-V`), sourced from `openfoia.__version__`. +- `openfoia analyze graph --no-text` — export the entity graph without + embedding document bodies. When text *is* included, the command now warns + that the export is plaintext and lives outside the encrypted database. +- `CHANGELOG.md` (this file) and a web-UI section in `docs/AIRGAP.md`. + +### Fixed + +- **Installer could mistake a checksum file for the binary.** `install.sh` + matched release assets by unanchored substring, so `pdf-extract-` + also matched `pdf-extract-.sha256`. With no guaranteed asset + ordering, the installer could have downloaded the checksum file and installed + it as the extractor. The match is now anchored to the exact asset name. This + sits on top of 4.0.0's fail-closed checksum verification. +- The benchmark's graph writer (`tests/benchmark_extraction.py`) spliced raw + JSON into an inline `` broke out and ran arbitrary + JavaScript on `file://`. `escape_json_for_script` now escapes `<`, `>`, `&`, + and U+2028/U+2029. Regenerate any `graph.html` produced before this release. +- **The web UI loaded JavaScript from a CDN.** `openfoia serve` pulled Tailwind + from `cdn.tailwindcss.com` on every page load, disclosing your IP and the + timing of your sessions. Replaced with a stylesheet served from disk; the web + UI now makes zero external requests, sends a restrictive Content-Security- + Policy header, rejects non-loopback `Host` headers, and keeps the auth token + out of the URL and the `Referer`. +- **Metadata, resource caps, and at-rest hardening.** Metadata stripping was + broadened, downloads and extraction gained size/resource caps, and database + file permissions were tightened. The gateways (email, fax, mail) were + hardened. +- **Installer verification fails closed.** `install.sh` verifies the + pdf-extract binary against a published `.sha256` and refuses to proceed when + it cannot verify, rather than warning and continuing. + +## [3.2.2] - 2026-04-10 + +### Added + +- LLM validation now reports keep/remove counts and surfaces errors instead of + failing quietly. + +### Changed + +- The extraction warning distinguishes a local AI provider from a cloud one, so + "AI is running" no longer reads the same whether or not documents are leaving + the machine. + +## [3.2.1] - 2026-04-10 + +### Added + +- Cassette-compatible LLM routing and selectable PDF extraction profiles. +- Multi-backend extraction pipeline with mention merging and an `--ensemble` + mode. +- LLM used as a validator rather than an extractor, plus junk filtering and + OCR-aware fuzzy merging (477 → 333 entities on the benchmark). +- `--model` flag and Qwen3 support. + +### Fixed + +- Crossref rate limiting, deduplication, error handling, progress output, and a + ProPublica 404. +- Forced re-extraction (`--force`). + +## [3.2.0] - 2026-03-24 + +### Added + +- DocumentCloud adapter and an interactive document reader in the graph view. +- Multi-layer MuckRock search with cleaner table display. +- MSG email support and file-type display in search results. + +### Fixed + +- spaCy auto-download. +- Web upload text extraction. (The upload path was later routed through the safe + ingest API and CSP-hardened in 4.0.0.) +- Portable-mode config, request-send persistence, and agent draft handling. + +## [3.1.1] - 2026-03-23 + +### Added + +- Python CI, a pre-commit hook running ruff lint and format, and `CLAUDE.md` + with the project's mission and principles. + +### Fixed + +- ruff lint and format across the codebase. + +## [3.1.0] - 2026-03-23 + +### Security + +- Duress mode redesigned: no stored password hash, an encrypted decoy database, + and opaque filenames. +- Honest security messaging, a written threat model, and install checksums. +- Addressed seven findings from an adversarial review. + +### Fixed + +- LLM-extracted entities are validated against the source text. +- MuckRock search uses tags (the API has no full-text search). +- The install script searches all releases for the pdf-extract binary. + +## [3.0.1] - 2026-03-22 + +### Changed + +- Core install is lightweight; heavy packages are opt-in extras. +- Every missing-dependency error now names the `openfoia install-extras` + command that fixes it. + +### Added + +- Portable install — the entire app lives on the USB stick. + +### Fixed + +- Install uses an isolated venv rather than polluting the system Python. + +## [3.0.0] - 2026-03-22 + +Baseline for this changelog. Earlier tags (`v0.0.1` through `v2.0.0`, +2026-02-19 to 2026-03-22) predate it; see the git history for details. + +[4.1.0]: https://github.com/JordanCoin/openfoia/compare/v4.0.0...v4.1.0 +[4.0.0]: https://github.com/JordanCoin/openfoia/compare/v3.2.2...v4.0.0 +[3.2.2]: https://github.com/JordanCoin/openfoia/compare/v3.2.1...v3.2.2 +[3.2.1]: https://github.com/JordanCoin/openfoia/compare/v3.2.0...v3.2.1 +[3.2.0]: https://github.com/JordanCoin/openfoia/compare/v3.1.1...v3.2.0 +[3.1.1]: https://github.com/JordanCoin/openfoia/compare/v3.1.0...v3.1.1 +[3.1.0]: https://github.com/JordanCoin/openfoia/compare/v3.0.1...v3.1.0 +[3.0.1]: https://github.com/JordanCoin/openfoia/compare/v3.0.0...v3.0.1 +[3.0.0]: https://github.com/JordanCoin/openfoia/releases/tag/v3.0.0 diff --git a/README.md b/README.md index d27872d..6afb7d5 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,20 @@ Your data never leaves your machine. Works offline. Works everywhere. ## Install +### With Claude Code (recommended) + +If you use [Claude Code](https://claude.ai/code), install the OpenFOIA plugin — you get the CLI plus a built-in copilot that knows every command: + +``` +/plugin marketplace add JordanCoin/openfoia +/plugin install openfoia@openfoia +/foia-install +``` + +The plugin ships a skill that teaches Claude how to run FOIA investigations, plus slash commands for the core loop: `/foia-search`, `/foia-investigate`, `/foia-graph`, `/foia-install`. The last one bootstraps the CLI on your machine. + +### Shell install (no Claude Code needed) + ```bash curl -fsSL https://raw.githubusercontent.com/JordanCoin/openfoia/main/install.sh | bash ``` @@ -50,7 +64,7 @@ openfoia crossref # check entities agai | **Encrypted storage** | SQLCipher AES-256. Decoy profile mode | | **Forensic purge** | 3-pass overwrite, shell history scrub, free space fill | | **Portable mode** | `openfoia portable` — everything stays on the USB, nothing on the host | -| **Metadata stripping** | Auto-strips EXIF, PDF author, DOCX revision history on ingest | +| **Metadata stripping** | Strips EXIF, PDF author, DOCX revision history on file and web-UI ingest. Email attachments and archived web pages are stored as received — see [THREAT_MODEL.md](docs/THREAT_MODEL.md) | ### PDF Extraction Engine diff --git a/docs/AIRGAP.md b/docs/AIRGAP.md index e3c2c39..6849ade 100644 --- a/docs/AIRGAP.md +++ b/docs/AIRGAP.md @@ -156,6 +156,25 @@ ollama serve & openfoia config --init # Select "ollama" as the AI provider ``` +## The Web UI Works Fully Offline + +`openfoia serve` needs no internet. It binds to `127.0.0.1`, serves a single +self-contained HTML page, and fetches nothing from any external host -- the +stylesheet is hand-written and inlined, there are no web fonts, no CDN +scripts, and no analytics. The only external URL anywhere on the page is a +link to the project's GitHub repo in the footer, which does nothing unless +you click it. + +This was not true before v4.0.0: the page loaded Tailwind CSS from +`cdn.tailwindcss.com`, so every page load made a DNS lookup and a TLS request +to a third party, and the UI was close to unreadable without one. If you are +running an older version on an air-gapped machine, expect a broken-looking +interface -- and on a networked machine, expect the request. Upgrade. + +Entity graphs (`openfoia analyze graph --view`) are self-contained too: a +single HTML file with inline CSS and JS that opens from `file://` with no +network access at all. + ## Security Checklist - [ ] Air-gapped machine has no WiFi/Ethernet/Bluetooth enabled @@ -165,4 +184,6 @@ openfoia config --init # Select "ollama" as the AI provider - [ ] Duress mode configured (`--duress-password`) - [ ] Swap disabled or encrypted on the air-gapped machine - [ ] Ollama running locally for AI features (no cloud API keys) +- [ ] Running v4.0.0 or later (earlier versions load CSS from a CDN on every + `openfoia serve` page load) - [ ] Physical security of the USB drive when not in use diff --git a/install.sh b/install.sh index b7f30c1..4d11c19 100755 --- a/install.sh +++ b/install.sh @@ -81,9 +81,18 @@ download_binary() { local url # Check all releases for binaries (they live on whichever release the - # glyph-api CI pushed them to — not necessarily the latest release) + # glyph-api CI pushed them to — not necessarily the latest release). + # + # Match the EXACT asset name: anchor on the leading '/' and the closing + # quote. Releases also carry a ".sha256" asset, and an unanchored + # substring match hits both — with JSON asset order not guaranteed, the + # installer could download the checksum file, chmod +x it, and install + # that as the binary. The API lists releases newest-first, so head -1 + # still picks the most recent release carrying this platform's binary. + local name_re + name_re=$(printf '%s' "$name" | sed 's/[][\.*^$/]/\\&/g') url=$(curl -fsSL "https://api.github.com/repos/${REPO}/releases" \ - | grep "browser_download_url.*${name}" \ + | grep -Eo "\"browser_download_url\"[[:space:]]*:[[:space:]]*\"[^\"]*/${name_re}\"" \ | head -1 \ | cut -d '"' -f 4) || true diff --git a/openfoia/__init__.py b/openfoia/__init__.py index c0364c7..0f19064 100644 --- a/openfoia/__init__.py +++ b/openfoia/__init__.py @@ -1,3 +1,3 @@ """OpenFOIA - Crowdsourced FOIA automation with AI-powered document analysis.""" -__version__ = "0.1.0" +__version__ = "4.1.0" diff --git a/openfoia/cli.py b/openfoia/cli.py index fbab200..49e79b9 100644 --- a/openfoia/cli.py +++ b/openfoia/cli.py @@ -27,6 +27,28 @@ ) +def _version_callback(value: bool) -> None: + if value: + from . import __version__ + + rprint(f"openfoia {__version__}") + raise typer.Exit() + + +@app.callback() +def _root( + version: bool = typer.Option( + False, + "--version", + "-V", + help="Show the installed OpenFOIA version and exit.", + callback=_version_callback, + is_eager=True, + ), +) -> None: + """Crowdsourced FOIA automation with AI-powered document analysis.""" + + # === Init Command === @@ -2571,16 +2593,27 @@ def analyze_graph( view: bool = typer.Option( False, "--view", "-v", help="Open interactive HTML visualization in browser" ), + no_text: bool = typer.Option( + False, + "--no-text", + help="Omit full document text from the export (keeps entities, links and context snippets)", + ), ): """Build entity relationship graph from extracted entities. Use --name to save graphs by investigation name. Each investigation gets its own graph that you can revisit later. + The exported .json and .html files are plaintext and live outside the + encrypted database. By default they embed the full text of every document + behind the graph, which is what makes the reader view work. Pass --no-text + to export the structure without the document bodies. + Examples: openfoia analyze graph --view # everything, open in browser openfoia analyze graph --name defense-contracts --view # save + view openfoia analyze graph --request REQ-001 --name epa # filter + save + openfoia analyze graph --no-text --name epa # structure only, no doc bodies openfoia analyze graphs # list saved graphs """ from .db import get_db_path, get_session @@ -2680,7 +2713,7 @@ def analyze_graph( "id": doc.id, "filename": doc.filename or "Unknown", "page_count": doc.page_count, - "text": doc.extracted_text or "", + "text": "" if no_text else (doc.extracted_text or ""), "request_id": doc.request_id, "source_url": source_url, } @@ -2709,6 +2742,13 @@ def analyze_graph( rprint(f" Relationships: {len(edges)}") if name: rprint(f" Saved as: [cyan]{name}[/cyan]") + if no_text: + rprint(" [dim]Document text omitted (--no-text). The reader view will be empty.[/dim]") + elif documents: + rprint( + "[yellow]Note:[/yellow] this export embeds full document text in plaintext, " + "outside the encrypted database — share deliberately, or re-run with --no-text." + ) if view: _generate_graph_html(graph_data, html_path) diff --git a/plugin/.claude-plugin/plugin.json b/plugin/.claude-plugin/plugin.json new file mode 100644 index 0000000..3291c4f --- /dev/null +++ b/plugin/.claude-plugin/plugin.json @@ -0,0 +1,10 @@ +{ + "name": "openfoia", + "description": "Help journalists and researchers run FOIA investigations using the openfoia CLI. Ships a skill covering the full records-search → download → OCR → extract → crossref → graph loop plus slash commands for each phase.", + "version": "0.1.0", + "author": { + "name": "Jordan Coin Jackson" + }, + "repository": "https://github.com/JordanCoin/openfoia", + "license": "MIT" +} diff --git a/plugin/README.md b/plugin/README.md new file mode 100644 index 0000000..7cc629d --- /dev/null +++ b/plugin/README.md @@ -0,0 +1,39 @@ +# OpenFOIA Claude Code plugin + +Ships a skill and four slash commands that make Claude a competent copilot for FOIA investigations using the [openfoia CLI](https://github.com/JordanCoin/openfoia). + +## What's in the box + +- **Skill** — `openfoia`: teaches Claude the full investigation loop (records search → download → OCR → extract → crossref → graph), when to use which extraction tier, privacy warnings before network calls, and how to read FOIA responses critically. +- **`/foia-install`** — install and initialize the `openfoia` CLI, with a warning before each step that touches the machine. +- **`/foia-search `** — multi-source search across MuckRock, OpenCorporates, SEC. +- **`/foia-investigate `** — end-to-end investigation on one request. +- **`/foia-graph `** — build or open a saved relationship graph. + +## Requires + +- `openfoia` CLI installed (`pip install -e ".[dev]"` from the openfoia repo, or `pip install openfoia`) +- For OCR: `openfoia install-extras ocr` + system `tesseract` and `poppler` +- For crossref's cloud sources: no API keys needed for public data + +## Privacy posture + +The skill teaches Claude to warn before any network call. All local operations (ingest, extract without cloud LLM, graph, purge) run entirely on-device. See `docs/THREAT_MODEL.md` in the openfoia repo for what is and isn't protected. + +## Install (local dev) + +From a clone of this repo: + +```bash +mkdir -p ~/.claude/plugins +ln -s "$(pwd)/plugin" ~/.claude/plugins/openfoia +``` + +Restart Claude Code. Check it loaded with `/help` — you should see the four `/foia-*` commands. + +## Install (marketplace) + +``` +/plugin marketplace add JordanCoin/openfoia +/plugin install openfoia@openfoia +``` diff --git a/plugin/commands/foia-graph.md b/plugin/commands/foia-graph.md new file mode 100644 index 0000000..5aceace --- /dev/null +++ b/plugin/commands/foia-graph.md @@ -0,0 +1,42 @@ +--- +allowed-tools: Bash(openfoia analyze graph:*), Bash(openfoia analyze graphs:*), Bash(openfoia request list:*), Bash(open:*) +description: Build or open an investigation graph from extracted entities +--- + +Build or open a relationship graph: $ARGUMENTS + +The argument can be: +- **A name of an existing saved graph** (e.g., `clearview-ai`) → open it +- **A request ID** (e.g., `INGEST-4F14FE` or `DC-25981836`) → build a new graph from that request +- **Empty or `list`** → list all saved graphs + +## Step-by-step + +1. **If the argument is empty or "list"**, show all saved graphs: + ```bash + openfoia analyze graphs + ``` + Then ask the user which one to open, or whether they want to build a new one. + +2. **If the argument looks like a saved-graph name** (matches one shown by `openfoia analyze graphs`), open it directly: + ```bash + open ~/.openfoia/graphs/.html + ``` + +3. **If the argument looks like a request ID** (starts with `DC-`, `REQ-`, `INGEST-`, etc.), build a new graph. Ask the user for a short descriptive name first — graphs without names aren't persisted interactively: + ```bash + openfoia analyze graph --request --name --view + ``` + +4. **If ambiguous**, run `openfoia analyze graphs` and `openfoia request list` in parallel to show both, then ask the user to clarify. + +## Output to surface + +After building: +- Entity count and relationship count (from the CLI output) +- File path to the saved HTML +- Confirm the browser opened + +After opening an existing graph, just confirm it's open — don't re-explain what pyvis is. + +Keep responses short. The graph is the output; your job is to get the user there. diff --git a/plugin/commands/foia-install.md b/plugin/commands/foia-install.md new file mode 100644 index 0000000..3056b45 --- /dev/null +++ b/plugin/commands/foia-install.md @@ -0,0 +1,69 @@ +--- +allowed-tools: Bash(command -v:*), Bash(which:*), Bash(openfoia:*), Bash(curl:*), Bash(pip install:*), Bash(pip3 install:*), Bash(ls:*), Bash(test:*), Bash(cat pyproject.toml) +description: Install or initialize the openfoia CLI — bootstraps everything the plugin needs to function +--- + +Install and initialize the openfoia CLI on this machine. + +**This command touches the user's machine** (downloads a binary, runs pip install, creates `~/.openfoia/`). Walk through each step, warn before each one, and stop if the user declines. + +## Step 1 — Check what's already there + +Run these in parallel to see what state we're in: + +```bash +command -v openfoia +test -f pyproject.toml && cat pyproject.toml | head -5 +test -d ~/.openfoia && ls ~/.openfoia +``` + +Based on the result, pick one of the branches below. + +## Branch A — openfoia is already installed + +If `command -v openfoia` prints a path: + +1. Confirm with the user: "openfoia is already at ``. Version is ``. Nothing to install." + - `--version` was added in 3.3.0. If it errors with "No such option", the install predates it — use `pip show openfoia | head -2` instead and mention that an upgrade is available. +2. If `~/.openfoia/` doesn't exist, offer to run `openfoia init` (creates the DB + loads 53 federal agencies). +3. Done. Suggest `/foia-search ` as the next step. + +## Branch B — we're in the openfoia repo (source install) + +If `pyproject.toml` exists in cwd and contains `name = "openfoia"`: + +1. Warn the user: "I'll install openfoia from this local repo with `pip install -e '.[dev]'`. This puts openfoia on your PATH in editable mode. OK?" +2. On approval, run: + ```bash + pip install -e ".[dev]" + ``` +3. Then initialize the DB: + ```bash + openfoia init + ``` +4. Confirm with `openfoia --version`. + +## Branch C — fresh install from the web + +If openfoia isn't installed and we're not in the repo: + +1. Warn the user: "I'll download and run https://raw.githubusercontent.com/JordanCoin/openfoia/main/install.sh. This will fetch a precompiled `pdf-extract` binary, pip-install openfoia, and create `~/.openfoia/`. OK to proceed?" +2. On approval, run: + ```bash + curl -fsSL https://raw.githubusercontent.com/JordanCoin/openfoia/main/install.sh | bash + ``` +3. Confirm with `openfoia --version`. + +## After any branch succeeds + +Tell the user what they can do next — succinctly: + +- `/foia-search ` — search MuckRock for existing FOIAs +- `/foia-investigate ` — full investigation loop +- `openfoia guide` — interactive quickstart + +If the install failed, surface the exact error and suggest either: +- `openfoia install-extras ocr` (if a later step needs OCR) +- Checking the openfoia repo README at https://github.com/JordanCoin/openfoia + +Keep the response short. The user mostly cares whether it worked, not the step-by-step. diff --git a/plugin/commands/foia-investigate.md b/plugin/commands/foia-investigate.md new file mode 100644 index 0000000..33e9011 --- /dev/null +++ b/plugin/commands/foia-investigate.md @@ -0,0 +1,64 @@ +--- +allowed-tools: Bash(openfoia:*) +description: Run the full FOIA investigation loop on a topic or MuckRock ID — search, download, OCR, extract, crossref, and graph +--- + +Run an end-to-end FOIA investigation on: $ARGUMENTS + +The argument can be either a topic (like `Palantir contracts`) or a MuckRock request ID (like `195614`). If it's numeric, treat it as a MuckRock ID and skip the search step. + +**Before running:** tell the user in plain language what network calls this will make — MuckRock for search/download, plus 10+ sources for crossref. Get explicit go-ahead. + +## Step-by-step + +1. **Search (if the argument is a topic, not an ID)** + ```bash + openfoia records search "$ARGUMENTS" --source muckrock --limit 10 + ``` + Show results, then ask the user which ID to investigate. Stop here until they pick one. + +2. **Download the PDFs** + ```bash + openfoia records download --source muckrock + ``` + List what came down. Flag any obvious boilerplate ("Responsive Records Attached", "Fee Waiver", etc.) vs. the likely substantive records. + +3. **Ingest** — ask the user which file to focus on, or offer to ingest all with `--recursive`: + ```bash + openfoia docs ingest downloads/.pdf + ``` + Note the document ID returned. + +4. **Try extract first** + ```bash + openfoia analyze extract + ``` + If entity count is very low (< 3 non-date entities) and the file is likely scanned, proceed to step 5. Otherwise skip to step 6. + +5. **OCR if needed** + ```bash + openfoia docs ocr downloads/.pdf -o /tmp/.txt + openfoia analyze extract --force + ``` + +6. **Read the extraction output critically.** Name any entity that looks like a false positive (keyword collision, OCR artifact). Explain what the document actually says vs. what the request asked for — the gap itself is often the story. + +7. **Crossref, scoped to this request** + Find the request ID with `openfoia request list`, then: + ```bash + openfoia crossref -r + ``` + Read the hits. Call out any flagged entities that appear in multiple sources — those are the investigative leads. + +8. **Build the graph** + ```bash + openfoia analyze graph --request --name --view + ``` + Pick a short, descriptive slug (e.g., `palantir-el-cajon`, `clearview-chicago`). `--view` opens the interactive HTML in the browser. + +9. **Summarize the finding.** Three things: + - What the user asked the agency for + - What the agency actually disclosed + - Whether the response was substantively responsive or a null-response-as-compliance pattern (agencies often search their vendor DB on a keyword and ship whatever matches, regardless of whether it relates) + +Keep the narrative tight and honest. If the investigation hit a dead end (wrong entity, no matches, scanned doc that OCR couldn't read), say so and suggest the next thread to pull. diff --git a/plugin/commands/foia-search.md b/plugin/commands/foia-search.md new file mode 100644 index 0000000..ae5de93 --- /dev/null +++ b/plugin/commands/foia-search.md @@ -0,0 +1,31 @@ +--- +allowed-tools: Bash(openfoia records search:*), Bash(openfoia agency search:*) +description: Search MuckRock, OpenCorporates, and SEC for existing records on a topic before filing a new FOIA +--- + +Search public records for the user's topic: $ARGUMENTS + +**Before running:** confirm with the user in one sentence that this hits external APIs (MuckRock, OpenCorporates, SEC). Only skip the warning if the user has already approved network use in this session. + +Follow these steps: + +1. Run MuckRock first — that's where completed FOIA responses live: + ```bash + openfoia records search "$ARGUMENTS" --source muckrock --limit 15 + ``` + +2. If the topic is a company/organization, also check OpenCorporates and SEC in parallel: + ```bash + openfoia records search "$ARGUMENTS" --source opencorporates --limit 10 + openfoia records search "$ARGUMENTS" --source sec --limit 10 + ``` + +3. Read the MuckRock results and group by angle (e.g., "police departments", "public health", "universities") so the user can pick a thread. + +4. If MuckRock returns 0 results, surface that clearly and suggest: + - Trying a broader or differently-phrased query + - Filing a new request via `/foia-investigate` or `openfoia request new` + +5. End by suggesting the next step — either `/foia-investigate ` to run the full loop on a specific result, or `openfoia records download ` to just pull the PDFs. + +Keep your output tight. Raw CLI tables are fine; don't re-format them into prose. diff --git a/plugin/skills/openfoia/SKILL.md b/plugin/skills/openfoia/SKILL.md new file mode 100644 index 0000000..f94004a --- /dev/null +++ b/plugin/skills/openfoia/SKILL.md @@ -0,0 +1,257 @@ +--- +name: openfoia +description: Use this skill when helping a journalist, researcher, or citizen run FOIA investigations using the openfoia CLI — searching public records (MuckRock, SEC EDGAR, OpenCorporates, DocumentCloud), downloading FOIA response PDFs, running OCR, extracting entities (people, orgs, dates, money), cross-referencing entities against investigative databases, building relationship graphs, or filing new FOIA requests. Trigger whenever the user mentions FOIA, public records, freedom of information, government transparency, document analysis for investigations, entity extraction from PDFs, Palantir/Clearview-style investigations, or specific openfoia commands. Also trigger when the user is in a directory where `openfoia` is installed and they ask about analyzing documents, even if they don't say "FOIA" explicitly. +--- + +# OpenFOIA investigation copilot + +You are assisting someone using OpenFOIA — a local-first, privacy-preserving FOIA automation tool — to investigate public records. The tool runs entirely on the user's machine unless they explicitly opt into network calls. Safety comes before speed. + +**If the `openfoia` command is not installed** (check with `command -v openfoia`), tell the user to run `/foia-install` before anything else. Don't try to install it yourself — `/foia-install` walks through the options with proper warnings. + +## Core mental model + +OpenFOIA is organized around one investigation loop: + +``` +records search → download → OCR → extract → crossref → graph +(external APIs) (PDFs) (if scanned) (entities) (confirm) (visualize) +``` + +Every step writes to a single SQLite database at `~/.openfoia/openfoia.db`. Document bytes live on disk at `~/.openfoia/docs/.pdf`. Graphs are saved HTML + JSON under `~/.openfoia/graphs/.html`. + +## Before running anything that touches the network + +Commands that leave the machine: `records search/fetch/download`, `crossref`, `request send`, `ingest --url ` (top-level web page ingest), `browse` (Tor routes to the target URL), and `analyze extract` when the configured AI provider is a cloud one (the provider comes from config, not from `--model`). Every other command is offline. + +Before running these, warn the user in one sentence — what's about to be sent where. Don't be preachy, just honest. Example: + +> "This hits MuckRock's API — your search query and the entity names go to their servers. OK to proceed?" + +If the user has already approved network use in this session, don't re-warn for the same source. + +## The investigation loop, step by step + +### 1. Records search — find existing FOIA responses before filing a new one + +```bash +openfoia records search "Palantir" --source muckrock --limit 15 +openfoia records search "Acme Corp" --source opencorporates -j us_ca +openfoia records search "Anthropic" --source sec --type 10-K +``` + +MuckRock is the primary source for completed FOIA responses (~150k requests). OpenCorporates is for company registration lookups. SEC is for US public company filings. The API has no full-text search on MuckRock — it falls back through tags → agency → user filters internally. + +**When to suggest filing a new request instead:** if search returns 0 results across sources, or if results are all old/irrelevant, pivot to `openfoia request new`. + +### 2. Download or fetch — grab the PDFs + +```bash +openfoia records download --source muckrock # download all files for a request +openfoia records fetch --source documentcloud # fetch full text of a single document into DB +openfoia ingest --url # ingest a web page as a document (network!) +``` + +`download` pulls all response documents to `./downloads/` — does NOT auto-ingest. `fetch` pulls a single document's text directly into the DB (useful when you just want text, not the PDF bytes); **`fetch` only supports `--source documentcloud`** — for MuckRock use `records download`. `openfoia ingest --url ` (top-level, distinct from `docs ingest`) scrapes a web page and stores it as a document — warn before running, it hits whatever URL you pass. + +Review filenames before ingest — FOIA responses often contain cover letter + actual records + boilerplate denials mixed in. + +### 3. Ingest — bring a document into OpenFOIA's storage + +```bash +openfoia docs ingest downloads/.pdf +openfoia docs ingest downloads/ --recursive +openfoia docs ingest ./doc.pdf -r REQ-2026-001 # associate with a request +``` + +Ingest automatically strips metadata (EXIF, author, producer, creation date). The `--keep-metadata` flag is available but should be used cautiously — stripping is the safe default, and some metadata can deanonymize the source. + +After ingest, each document gets a UUID. The short form (first 8 chars) works for most commands. + +### 4. OCR — only if the PDF is scanned + +```bash +openfoia docs ocr downloads/.pdf -o /tmp/out.txt +``` + +If `analyze extract` returns very few entities and the PDF is scanned (images of text, not text), OCR first. Requires `openfoia install-extras ocr` plus system-level `tesseract` and `poppler`. + +### 5. Extract — pull entities from document text + +```bash +openfoia analyze extract # default: LLM-validated +openfoia analyze extract --ensemble # run all NER backends +openfoia analyze extract --model # provider comes from config, not this flag +openfoia analyze extract --force # re-extract +``` + +Four-tier fallback pipeline: LLM → GLiNER → spaCy → regex. Regex always works. Each tier is better than the one below it, but all have cost tradeoffs. The LLM validation step keeps/removes entities based on document context (e.g., will keep "SOUTH BAY FOUNDRY, INC" as a real organization but flag an OCR artifact as junk). + +**Entity types extracted:** person, organization, location, date, money, document_id, phone, email, address. + +### 6. Crossref — check entities against investigative databases + +```bash +openfoia crossref -r # scope to one request +openfoia crossref -d # scope to one document +openfoia crossref # everything (can be slow) +openfoia crossref --sources icij # offline only, needs ICIJ CSVs downloaded +``` + +Sources include MuckRock, OpenCorporates, SEC EDGAR, DocumentCloud, OpenSanctions, ICIJ Offshore Leaks, USAspending, FEC, govinfo, regulations.gov. This is the equivalent of Maltego's paid service but free and local-first. + +**Always scope crossref** to a request or document unless the user explicitly wants to re-check everything. Full-DB crossref hammers 10+ APIs for every entity in the database. + +Crossref output flags entities that appear in multiple sources — that's the interesting signal (e.g., a vendor named in a FOIA response AND listed as a government contractor AND appearing on a sanctions list). + +### 7. Graph — visualize the relationships + +```bash +openfoia analyze graph --view # everything +openfoia analyze graph --request --name my-investigation --view +openfoia analyze graph --campaign --name campaign-graph --view +openfoia analyze graph --request --name my-investigation --no-text # omit document bodies +openfoia analyze graphs # list saved +``` + +Graphs render as a single self-contained HTML file (inline CSS/JS, no external dependencies, no network access) saved to `~/.openfoia/graphs/.html`. Use `--name` to save — unnamed graphs export to `graph.json` in the cwd, and `--view` writes the HTML alongside it. + +**Tell the user what a graph file contains.** By default the export embeds the full extracted text of every document, in plaintext, outside the encrypted database — so the HTML/JSON is the whole investigation in one shareable file. Pass `--no-text` to export just the entities and relationships. The command prints a one-line reminder when text is included; don't let the user email a graph without knowing what's in it. + +## How to read the output of a run + +- **"LLM validation: keep=N, remove=M"** — the LLM validator kept N entities as real and removed M as junk. Big removal numbers usually mean OCR noise or a regex false positive. +- **"from N entities"** — the raw extraction count before validation. +- **Confidence < 70%** — eyeball it. Often real, sometimes junk. +- **"No entities found"** on a non-scanned PDF — likely a parser failure, not a document problem. Try `--ensemble`. + +## Filing a new request + +```bash +openfoia agency search "Police Department" -n 30 # -n is the result limit; there is no state filter +openfoia request new -a -s "subject" -f request-body.txt -n "Full Name" -e "you@email" +openfoia request send -a -s "subject" -b "body text" -n "Full Name" -e "you@email" + +``` + +Use `openfoia template list` for pre-written FOIA boilerplate. Always double-check the delivery method (`email`, `fax`, `mail`) matches the agency's preference — `agency info ` shows `preferred_method`. + +## Campaigns — crowdsource the same request + +```bash +openfoia campaign create -n "Palantir Contracts 2026" -d "Contracts and comms" -t body.txt --organizer "Jane Doe" -e jane@example.com --target 500 +openfoia campaign distribute # assigns requests to participants +openfoia campaign progress # per-participant status grid +``` + +Campaigns let one investigation run against hundreds of agencies. Matches MuckRock's Assignments model but runs locally. + +## Honest limits worth calling out + +- **No versioned extraction runs.** `--force` re-extract overwrites entities. There's no audit trail of "what did Claude-Haiku say last week vs. Claude-Opus today." Tracked in issue #63. +- **DB is NOT encrypted by default.** Users must run `openfoia install-extras encryption` to enable SQLCipher. +- **"Purge" means purge.** `openfoia purge` deletes everything and is not reversible. Confirm twice. It is a normal delete unless `--secure` is passed, and even then SSD wear-levelling means old blocks may survive — say so rather than promising erasure. +- **Optional deps are lazy-imported.** If the user hits `ModuleNotFoundError`, point them to `openfoia install-extras ` rather than digging into pip. + +## Tone and posture + +The people using this tool are investigating power — often under real pressure, sometimes in hostile environments. Be direct, honest about limits, and skeptical of false positives. When in doubt about a warning, issue it. When a FOIA response looks like a "null response dressed as compliance" (e.g., the agency searched for "Foundry" as a vendor name and returned a metal shop), call it out explicitly — that pattern matters for the investigation. + +## Other subsystems worth knowing + +These aren't part of the core loop but come up often enough to keep in mind. + +### Custom entity types — extend what the extractor looks for +```bash +openfoia entities list # show built-in + custom types +openfoia entities add --name vessel --pattern "IMO \d{7}" # regex-based +openfoia entities import my-types.csv # bulk import (CSV or Excel, NOT JSON) +openfoia entities test -t "" # dry-run all types against sample text (also -f or stdin) +``` +Custom types slot into the regex fallback tier — useful for domain-specific patterns (ship IMOs, case numbers, contract IDs) that general NER misses. + +### Deadlines — statutory FOIA response tracking +```bash +openfoia deadlines list # all open requests with due dates +openfoia deadlines check # flag overdue requests +``` +Due dates are computed from `Agency.typical_response_days` at send time. Federal FOIA default is 20 business days. + +### Templates — FOIA boilerplate +```bash +openfoia template list # available templates +openfoia template generate standard -a FBI -s "Records on X" -n "Your Name" -e you@example.com -o draft.txt +openfoia template exemptions # reference for b(1)-b(9) exemption codes +``` +Templates include contract requests, communications, personnel records, etc. Use `exemptions` when reading denials — agencies often cite b(5) (deliberative process) or b(7)(A) (law enforcement) to withhold. + +### FollowTheMoney (FTM) export/import — integrate with Aleph/OCCRP +```bash +openfoia analyze export -o investigation.ftm.json +openfoia analyze import aleph-export.ftm.json +``` +FTM is the open standard for investigative data exchange. Export puts OpenFOIA's entities + relationships into the format Aleph, OCCRP's stack, and most investigative tooling reads natively. + +### Web UI — local server with token auth +```bash +openfoia serve # binds a random free port; prints the URL + token +``` +Bound to localhost only. The default port is `0`, meaning a random free port — there is no fixed 8000. On start it prints the full URL including a `?token=...`; that token is required. The page loads nothing from the network. + +### Privacy tools + +```bash +openfoia egress-status # show whether traffic goes DIRECT or via Tor, honestly +openfoia egress-status --tor # check the Tor mode instead of the configured default +openfoia portable # move all data beside the binary (USB-safe) +openfoia browse # visit a URL with Tor routing + fingerprint hardening +openfoia purge # delete everything (irreversible) +openfoia purge --secure # 3-pass overwrite before deleting (opt-in) +``` +`egress-status` reports the current network egress policy — DIRECT vs TOR, whether the Tor SOCKS proxy is actually reachable right now, and exactly what is and isn't protected (see `docs/THREAT_MODEL.md`). Run it before any command that hits the network. `portable` is for journalists moving between machines — the DB and docs travel with the binary, no traces in `~/.openfoia`. `browse` uses the Tor SOCKS proxy (user must have tor running). Plain `purge` is an ordinary delete; multi-pass overwriting is opt-in via `--secure` (and `--fill` for free space). Note that on SSDs overwriting does not reliably erase the old blocks — full-disk encryption is the real protection. Don't promise more than that. + +### Config, setup, migration +```bash +openfoia init # first-time DB setup +openfoia guide # interactive quickstart +openfoia config --init # interactive first-time config +openfoia config --show # print current config (also via OPENFOIA_* env or ~/.openfoia/config.json) +openfoia install-extras # ner, ocr, fax, mail, cloud-ai, tor, browser, encryption, all +openfoia db upgrade # run Alembic migrations +openfoia db encrypt --password # convert plaintext DB to SQLCipher (AES-256) +``` + +### Request and agency lookup (alongside the core loop) +```bash +openfoia request list # all requests, status, days pending +openfoia request status # detailed timeline of one request +openfoia agency list # all agencies in the DB +openfoia agency info # contact info + preferred delivery method + stats +openfoia campaign list # all campaigns +openfoia campaign status # progress summary +openfoia campaign join -n "Your Name" -e you@example.com # opt into someone else's campaign +``` + +## Quick reference + +| I want to... | Command | +|-----|-----| +| Find existing FOIAs on a topic | `openfoia records search "" --source muckrock` | +| Download response PDFs | `openfoia records download --source muckrock` | +| Add PDFs to the database | `openfoia docs ingest ` | +| OCR a scanned PDF | `openfoia docs ocr -o ` | +| Pull entities from a doc | `openfoia analyze extract ` | +| Verify entities exist elsewhere | `openfoia crossref -r ` | +| Build/view a graph | `openfoia analyze graph --name --view` | +| List saved graphs | `openfoia analyze graphs` | +| Export to FollowTheMoney | `openfoia analyze export -o .ftm.json` | +| File a new request | `openfoia request new -a -s "..." -f body.txt -n "Name" -e you@example.com` | +| See all my requests | `openfoia request list` | +| Start a campaign | `openfoia campaign create -n "..." -d "..." -t body.txt --organizer "..." -e you@example.com` | +| Check FOIA deadlines | `openfoia deadlines list` | +| Find an agency | `openfoia agency search ""` | +| Add custom entity type | `openfoia entities add --name --pattern ""` | +| Start the web UI | `openfoia serve` (localhost only, token auth) | +| Enable USB portable mode | `openfoia portable` | +| Encrypt the database | `openfoia install-extras encryption && openfoia db encrypt --password ` | +| Purge everything | `openfoia purge` (irreversible; `--secure` for overwrite) | diff --git a/pyproject.toml b/pyproject.toml index 68f9a28..a1a9289 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "openfoia" -version = "4.0.0" +version = "4.1.0" description = "Local-first investigation toolkit for journalists, researchers, and citizens" readme = "README.md" license = "AGPL-3.0" @@ -62,6 +62,9 @@ dev = [ "pytest-cov>=4.1.0", "ruff>=0.1.0", "mypy>=1.8.0", + # tests/test_plugin.py parses plugin frontmatter. Test-only -- keep it out + # of core deps. + "pyyaml>=6.0", ] [project.scripts] diff --git a/tests/benchmark_extraction.py b/tests/benchmark_extraction.py index 2ce7846..1ca20b4 100644 --- a/tests/benchmark_extraction.py +++ b/tests/benchmark_extraction.py @@ -16,6 +16,7 @@ from pathlib import Path from openfoia.config import load_config +from openfoia.graph_template import escape_json_for_script from openfoia.pipeline.extract import ( EntityExtractor, _gliner_available, @@ -198,8 +199,11 @@ def generate_graph_html(result, output_path): graph_data = json.dumps({"nodes": nodes, "edges": edges}) - # Write the HTML file with embedded graph data - output_path.write_text(_GRAPH_HTML_TEMPLATE.replace("__GRAPH_DATA__", graph_data)) + # Write the HTML file with embedded graph data. The JSON is spliced straight + # into a ` broke out and executed on `file://`. v4.0.0's + escape_json_for_script escapes `<`, `>`, `&` and U+2028/U+2029. +- C6: the web UI must load nothing from a CDN, so merely opening it never + phones a third party. +""" + +from __future__ import annotations + +import json +import re + +import pytest + +from openfoia.graph_template import escape_json_for_script, render +from openfoia.server import get_index_html + +# A document body that breaks out of a " + + +def _render_graph(tmp_path, doc_text): + graph_data = { + "nodes": [{"id": "e1", "label": "Acme Corp", "type": "organization"}], + "edges": [], + "documents": { + "d1": { + "id": "d1", + "filename": "response.pdf", + "page_count": 1, + "text": doc_text, + "request_id": "REQ-001", + "source_url": None, + } + }, + } + out = tmp_path / "graph.html" + render(json.dumps(graph_data), out) + return out.read_text() + + +class TestGraphScriptInjection: + """C7 -- graph HTML must not let document text break out of ` + # after it must be the template's own closing tag. + start = html.index("var graphData =") + data_block = html[start : html.index("", start)] + + assert "" not in data_block + # If the payload text appears, its opening `<` must be \u003c-escaped so + # it cannot start or close a tag. v4.0.0 escapes both `<` and `>`. + assert "window.PWNED" not in data_block or "\\u003c/script" in data_block + # The literal breakout sequence must be neutralized wherever it appears. + assert PAYLOAD not in html + # No raw `<` survives in the embedded data itself -- every one is + # \u003c-escaped, which also kills `", + "bare and markup", + "quote \" and apostrophe ' and backslash \\", + "line sep \u2028 and paragraph sep \u2029 mid-text", + "Prince George's County v. O'Brien", + ], + ) + def test_hostile_and_awkward_text_round_trips(self, tmp_path, text): + html = _render_graph(tmp_path, text) + start = html.index("var graphData =") + raw = html[start + len("var graphData =") : html.index("\n", start)].strip().rstrip(";") + + assert json.loads(raw)["documents"]["d1"]["text"] == text + # Raw U+2028/U+2029 would be an illegal line terminator in older JS. + assert "\u2028" not in raw + assert "\u2029" not in raw + + def test_escape_helper_is_a_no_op_on_benign_json(self): + benign = json.dumps({"text": "Department of Justice, 441 G Street NW"}) + assert escape_json_for_script(benign) == benign + + def test_graph_html_loads_nothing_from_the_network(self, tmp_path): + html = _render_graph(tmp_path, "harmless text") + assert not re.findall(r'(?:src|href)\s*=\s*"https?://', html) + + +class TestWebUIIsOffline: + """C6 -- `openfoia serve` must make zero external requests.""" + + def test_no_external_resources_are_loaded(self): + html = get_index_html() + loaded = re.findall(r'(?:src|href)\s*=\s*"(https?://[^"]*)"', html) + # An the user must click is fine; a loaded resource is not. + anchors = re.findall(r']*href="(https?://[^"]*)"', html) + assert [u for u in loaded if u not in anchors] == [] + + def test_no_cdn_script_tag(self): + html = get_index_html() + assert "cdn.tailwindcss.com" not in html + assert not re.findall(r"]*\ssrc\s*=", html)