From 07649bdb68bcf66f47111766c2d1590b0c90501d Mon Sep 17 00:00:00 2001 From: Vasily Ilin Date: Sat, 12 Sep 2026 16:01:54 -0700 Subject: [PATCH] Restore estimated dollar costs for Azure GPT-6 reviews --- AGENTS.md | 2 +- README.md | 2 +- python/azure-review.md | 9 ++++- python/lean_pool/review.py | 47 +++++++++++++++++++---- python/tests/test_codex_review.py | 64 ++++++++++++++++++++++++++++++- 5 files changed, 111 insertions(+), 13 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 346b87ac46..d8e82400c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,7 +22,7 @@ Lean Pool sits between [`mathlib`](https://github.com/leanprover-community/mathl So far, projects have been added by hand: each is a suitable, permissively licensed (Apache-2.0 or MIT) Lean repository, bumped to the latest Lean and Mathlib, made to pass [CI](.github/workflows/lean_action_ci.yml) — it builds warning-free and clears Mathlib's linters, the style checker, and the repository quality gates (no `sorry`/`admit`, no axioms beyond `Classical.choice`/`propext`/`Quot.sound`, no `unsafe`/`partial`, file headers, size limits) — and an [LLM review](.github/REVIEW_RULES.md) of fit and significance, then merged. -LLM reviews use GPT-6-Astra at `xhigh` reasoning effort through the Azure VM's Codex account pool, without paid OpenAI API requests or an API fallback. See [review operations](python/azure-review.md) for deployment and diagnostics. +LLM reviews use GPT-6-Astra at `xhigh` reasoning effort through the Azure VM's Codex account pool, without paid OpenAI API requests or an API fallback. Reviews also show an estimated dollar cost at official Standard API token rates, labeled separately from Codex quota billing. See [review operations](python/azure-review.md) for deployment and diagnostics. Project PRs also receive an advisory Greptile review, configured in [`.greptile/`](.greptile/), for cross-file integration, reusable abstractions, completeness, maintainability, and measured cost. It supplements rather than replaces the independent LLM verdict. diff --git a/README.md b/README.md index 47e487854e..47a83fc36f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Semantic search is also available via the [API](https://search.octo.axiomatic-ai So far, projects have been added by hand: each is a suitable, permissively licensed (Apache-2.0 or MIT) Lean repository, bumped to the latest Lean and Mathlib, made to pass [CI](.github/workflows/lean_action_ci.yml) — it builds warning-free and clears Mathlib's linters, the style checker, and the repository quality gates (no `sorry`/`admit`, no axioms beyond `Classical.choice`/`propext`/`Quot.sound`, no `unsafe`/`partial`, file headers, size limits) — and an [LLM review](.github/REVIEW_RULES.md) of fit and significance, then merged. -LLM reviews use GPT-6-Astra at `xhigh` reasoning effort through the Azure VM's Codex account pool, without paid OpenAI API requests or an API fallback. See [review operations](python/azure-review.md) for deployment and diagnostics. +LLM reviews use GPT-6-Astra at `xhigh` reasoning effort through the Azure VM's Codex account pool, without paid OpenAI API requests or an API fallback. Reviews also show an estimated dollar cost at official Standard API token rates, labeled separately from Codex quota billing. See [review operations](python/azure-review.md) for deployment and diagnostics. Project PRs also receive an advisory Greptile review, configured in [`.greptile/`](.greptile/), for cross-file integration, reusable abstractions, completeness, maintainability, and measured cost. It supplements rather than replaces the independent LLM verdict. diff --git a/python/azure-review.md b/python/azure-review.md index 7ac5d95cb0..035ea8a2ba 100644 --- a/python/azure-review.md +++ b/python/azure-review.md @@ -4,7 +4,14 @@ The LLM review workflow uses `gpt-6-astra` at `xhigh` reasoning effort through the authenticated Codex account pool on the `lean` Azure VM. It consumes that pool's Codex quota, with no OpenAI API key and no fallback to paid API requests. The existing review rubrics, CI prerequisite, verdict aggregation, and sticky -comments still apply. Comments identify the model, token counts, and quota billing. +comments still apply. Comments identify the model, token counts, quota billing, +and an **estimated cost** at the model's [official Standard API rates](https://developers.openai.com/api/docs/pricing). +For GPT-6-Astra, these are $10/M input tokens and $50/M output tokens; requests +above 272,000 input tokens use $20/M input and $75/M output (verified 2026-09-12). +The estimate values all input at the uncached rate, excluding cache-write +premiums, tool fees, and VM costs. It is a nominal API equivalent, not an invoice +for Codex usage. Each rubric is priced separately before summing its cost, so +five short prompts do not accidentally incur the long-context rate. GitHub Actions continues to fetch the PR and post the comment. Only the review instructions and contributor text cross SSH. The workflow checks out its trusted diff --git a/python/lean_pool/review.py b/python/lean_pool/review.py index 18931bc4be..a755be8bd9 100644 --- a/python/lean_pool/review.py +++ b/python/lean_pool/review.py @@ -140,6 +140,11 @@ # Source: https://developers.openai.com/api/docs/pricing — update when # bumping DEFAULT_MODEL or when OpenAI changes pricing. PRICING_PER_M: dict[str, dict[str, tuple[tuple[float, float], tuple[float, float]]]] = { + # Official Standard API rates verified 2026-09-12. Codex has no invoice + # amount, so its displayed estimate uses these uncached token rates. + "gpt-6-astra": { + "standard": ((10.00, 50.00), (20.00, 75.00)), + }, # The `gpt-5.6` key also covers the alias itself, which routes to # gpt-5.6-sol; both rows carry Sol rates. "gpt-5.6-sol": { @@ -165,7 +170,7 @@ def pricing_rates( Args: model: Model name as reported by the API — a dated snapshot like ``gpt-5.6-sol-2026-06-17`` matches its family prefix. - tier: Service tier the request was billed at. + tier: Service tier, or ``codex-azure`` for a Standard API estimate. input_tokens: Prompt size, which selects the short- or long-context rate. @@ -173,12 +178,19 @@ def pricing_rates( The matching rate pair, or ``None`` when the model/tier pair is not in :data:`PRICING_PER_M`. """ + if tier == codex_review.TIER: + tier = "standard" for prefix in sorted(PRICING_PER_M, key=len, reverse=True): if model == prefix or model.startswith(prefix): rates = PRICING_PER_M[prefix].get(tier) if rates is None: return None - return rates[1] if input_tokens >= LONG_CONTEXT_INPUT_TOKENS else rates[0] + long_context = ( + input_tokens > LONG_CONTEXT_INPUT_TOKENS + if prefix == "gpt-6-astra" + else input_tokens >= LONG_CONTEXT_INPUT_TOKENS + ) + return rates[1] if long_context else rates[0] return None @@ -1267,13 +1279,23 @@ def run_project_rubrics( return outcomes +def render_estimated_cost(cost: float) -> str: + """Label the nominal token value without implying an actual Codex charge.""" + return ( + f"**Estimated cost:** ${cost:.4f} " + "([Standard API equivalent](https://developers.openai.com/api/docs/pricing); " + "uncached input)" + ) + + def render_usage(usage: Any, model: str, tier: str, effort: str | None = None) -> str: """Render a one-line token / tier / effort / cost footer. Returns an empty string if ``usage`` is unavailable. Cost is computed from :data:`PRICING_PER_M` at the short- or long-context rate the request's input size lands in, and suppressed when the model/tier - pair is not listed there. + pair is not listed there. Azure uses an explicitly labeled Standard + API equivalent, valuing all input tokens at the uncached rate. """ if usage is None: return ( @@ -1289,13 +1311,18 @@ def render_usage(usage: Any, model: str, tier: str, effort: str | None = None) - parts.append(f"**Effort:** `{effort}`") if tier == codex_review.TIER: parts.append("**Billing:** Codex account quota on Azure (no API credits)") - return " · ".join(parts) rates = pricing_rates(model, tier, in_tok) if rates is not None: in_price, out_price = rates cost = (in_tok * in_price + out_tok * out_price) / 1_000_000 - cost_cell = f"**Cost:** ${cost:.4f}" - if in_tok >= LONG_CONTEXT_INPUT_TOKENS: + cost_cell = ( + render_estimated_cost(cost) + if tier == codex_review.TIER + else f"**Cost:** ${cost:.4f}" + ) + if in_tok > LONG_CONTEXT_INPUT_TOKENS or ( + in_tok == LONG_CONTEXT_INPUT_TOKENS and not model.startswith("gpt-6-astra") + ): cost_cell += " (long-context rate)" parts.append(cost_cell) else: @@ -1511,8 +1538,12 @@ def _render_rubric_usage(outcomes: list[RubricOutcome], effort: str | None) -> s parts.append(f"**Effort:** `{effort}`") if all(o.result.tier == codex_review.TIER for o in outcomes): parts.append("**Billing:** Codex account quota on Azure (no API credits)") - elif cost > 0 or not unpriced: - cost_cell = f"**Cost:** ${cost:.4f}" + if cost > 0 or not unpriced: + cost_cell = ( + render_estimated_cost(cost) + if any(o.result.tier == codex_review.TIER for o in outcomes) + else f"**Cost:** ${cost:.4f}" + ) if unpriced: cost_cell += " (partial — some calls unpriced)" parts.append(cost_cell) diff --git a/python/tests/test_codex_review.py b/python/tests/test_codex_review.py index ea0a516d44..5675d908fe 100644 --- a/python/tests/test_codex_review.py +++ b/python/tests/test_codex_review.py @@ -61,7 +61,9 @@ def ssh(command, **kwargs): assert result.usage.prompt_tokens == 20 footer = review.render_usage(result.usage, result.model, result.tier, result.effort) assert "no API credits" in footer - assert "$" not in footer + assert "**Estimated cost:** $0.0007" in footer + assert "Standard API equivalent" in footer + assert "uncached input" in footer @pytest.mark.parametrize( @@ -182,7 +184,7 @@ def communicate(self, *args, **kwargs): def test_azure_rubric_footer_reports_quota(): - """Five-rubric reviews report quota billing without inventing a dollar cost.""" + """Five-rubric reviews report token value separately from quota billing.""" result = review.ReviewResult( {}, SimpleNamespace(prompt_tokens=20, completion_tokens=10), @@ -195,4 +197,62 @@ def test_azure_rubric_footer_reports_quota(): footer = review._render_rubric_usage(outcomes, "xhigh") assert "100 in / 50 out" in footer assert "no API credits" in footer + assert "**Estimated cost:** $0.0035" in footer + + +@pytest.mark.parametrize( + "input_tokens,expected", + [(100_000, "$1.5000"), (272_000, "$3.2200"), (272_001, "$6.1900")], +) +def test_azure_estimate_uses_official_context_rates(input_tokens, expected): + """The full request switches rates only above the official 272K boundary.""" + usage = SimpleNamespace(prompt_tokens=input_tokens, completion_tokens=10_000) + footer = review.render_usage(usage, "gpt-6-astra", codex_review.TIER) + assert f"**Estimated cost:** {expected}" in footer + assert "no API credits" in footer + assert "**Cost:**" not in footer + + +def test_azure_rubrics_price_each_request_before_summing(): + """A large aggregate of short prompts does not trigger long-context pricing.""" + result = review.ReviewResult( + {}, + SimpleNamespace(prompt_tokens=100_000, completion_tokens=1_000), + codex_review.TIER, + None, + "gpt-6-astra", + "xhigh", + ) + footer = review._render_rubric_usage([SimpleNamespace(result=result)] * 5, "xhigh") + assert "500,000 in / 5,000 out" in footer + assert "**Estimated cost:** $5.2500" in footer + + +def test_azure_missing_usage_does_not_claim_zero_cost(): + """Unavailable token counts are never represented as a free review.""" + footer = review.render_usage(None, "gpt-6-astra", codex_review.TIER) + assert "$" not in footer + assert "no API credits" in footer + result = review.ReviewResult({}, None, codex_review.TIER, None, "gpt-6-astra", None) + footer = review._render_rubric_usage([SimpleNamespace(result=result)], None) assert "$" not in footer + + +def test_azure_partial_usage_labels_partial_estimate(): + """An incomplete rubric total cannot masquerade as the complete cost.""" + known = review.ReviewResult( + {}, + SimpleNamespace(prompt_tokens=100_000, completion_tokens=1_000), + codex_review.TIER, + None, + "gpt-6-astra", + None, + ) + missing = review.ReviewResult( + {}, None, codex_review.TIER, None, "gpt-6-astra", None + ) + footer = review._render_rubric_usage( + [SimpleNamespace(result=known), SimpleNamespace(result=missing)], None + ) + assert "**Estimated cost:** $1.0500" in footer + assert "partial" in footer