diff --git a/AGENTS.md b/AGENTS.md index cd0fc91..83988aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ Tests are in `tests/` with shared fixtures in `tests/conftest.py`. Supporting ma - `uv run python scripts/smoke_harness.py --budget 1` — run CLI smoke harness. - `uv run python scripts/score_check.py` — score regression check. - `uv run python scripts/score_check.py --update` — update baselines after improvement. -- `uv run python scripts/live_integration.py --phase green --artifact FILE --model openai/gpt-4o-mini --budget 15 --objective "..." --evaluator-command bash evaluators/eval.sh` — GREEN phase optimization. +- `uv run python scripts/live_integration.py --phase green --artifact FILE --model openai/gpt-5.6-sol --budget 15 --objective "..." --evaluator-command bash evaluators/eval.sh` — GREEN phase optimization. ## Coding Style & Naming Conventions Target Python is `>=3.10`. Follow existing style: diff --git a/CLAUDE.md b/CLAUDE.md index e641cc1..530512a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,20 +10,20 @@ uv run pytest uv run optimize-anything --help # Optimize -uv run optimize-anything optimize seed.txt --evaluator-command bash eval.sh --model openai/gpt-4o-mini --objective "Improve quality" +uv run optimize-anything optimize seed.txt --evaluator-command bash eval.sh --model openai/gpt-5.6-sol --objective "Improve quality" # Generate evaluator (default type: judge) uv run optimize-anything generate-evaluator seed.txt --objective "Score quality" > eval.py # Score one artifact -uv run optimize-anything score artifact.txt --judge-model openai/gpt-4o-mini --objective "Score clarity" +uv run optimize-anything score artifact.txt --judge-model openai/gpt-5.6-luna --objective "Score clarity" # Analyze for quality dimensions -uv run optimize-anything analyze artifact.txt --judge-model openai/gpt-4o-mini --objective "Quality" +uv run optimize-anything analyze artifact.txt --judge-model openai/gpt-5.6-luna --objective "Quality" # Validate across providers uv run optimize-anything validate artifact.txt \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 google/gemini-2.0-flash \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score quality" \ --intake-file intake.json ``` @@ -70,6 +70,8 @@ uv run optimize-anything validate artifact.txt \ Notes: - Exactly one evaluator source: `--evaluator-command` OR `--evaluator-url` OR `--judge-model`. - Early stop auto-activates when budget > 30. +- The proposer defaults to `openai/gpt-5.6-sol` after checking `--model` and `OPTIMIZE_ANYTHING_MODEL`; generated judge and composite evaluators default to `openai/gpt-5.6-luna`. +- LLM judge calls use each provider's sampling defaults unless an explicit temperature is supplied. ## Validate subcommand @@ -77,11 +79,13 @@ Notes: ```bash uv run optimize-anything validate runs/run-20260303-130000/best_artifact.txt \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 google/gemini-2.0-flash \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score clarity, constraints, and robustness" \ --intake-file intake.json ``` +Gemini uses LiteLLM's `gemini/` provider prefix. + ## Plugin structure ```text diff --git a/EXAMPLES.md b/EXAMPLES.md index 67696ef..d803835 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -36,7 +36,7 @@ Optimization output examples should follow this structure: ```bash optimize-anything optimize parse_duration.py \ --evaluator-command bash eval_tests.sh \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --objective "Pass all duration parsing tests" \ --budget 20 ``` @@ -61,7 +61,7 @@ Sample output excerpt: ```bash optimize-anything optimize error_template.txt \ --evaluator-url http://localhost:8080/evaluate \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --objective "Make API error messages clear and actionable" \ --budget 15 ``` @@ -72,9 +72,9 @@ optimize-anything optimize error_template.txt \ ```bash optimize-anything optimize support_prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Improve clarity, constraints, and tone" \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 30 ``` @@ -84,10 +84,10 @@ optimize-anything optimize support_prompt.txt \ ```bash optimize-anything optimize prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Generalize across user intents" \ --dataset data/train.jsonl \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 120 \ --workers 6 \ --cache --run-dir runs @@ -101,11 +101,11 @@ With validation set: ```bash optimize-anything optimize prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Generalize to unseen examples" \ --dataset data/train.jsonl \ --valset data/val.jsonl \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 150 \ --cache --cache-from runs/run-20260303-120000 \ --run-dir runs @@ -117,11 +117,13 @@ optimize-anything optimize prompt.txt \ ```bash optimize-anything validate runs/run-20260303-130000/best_artifact.txt \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 google/gemini-2.0-flash \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score clarity, correctness, and robustness" \ --intake-file intake.json ``` +Gemini uses LiteLLM's `gemini/` provider prefix. + Sample output excerpt: ```json @@ -129,9 +131,9 @@ Sample output excerpt: "artifact_file": "runs/run-20260303-130000/best_artifact.txt", "objective": "Score clarity, correctness, and robustness", "providers": [ - {"provider": "openai/gpt-4o-mini", "score": 0.82, "reasoning": "..."}, - {"provider": "anthropic/claude-sonnet-4-5", "score": 0.79, "reasoning": "..."}, - {"provider": "google/gemini-2.0-flash", "score": 0.81, "reasoning": "..."} + {"provider": "openai/gpt-5.6-luna", "score": 0.82, "reasoning": "..."}, + {"provider": "anthropic/claude-sonnet-5", "score": 0.79, "reasoning": "..."}, + {"provider": "gemini/gemini-3.6-flash", "score": 0.81, "reasoning": "..."} ], "mean": 0.8066666667, "stddev": 0.0152752523, @@ -147,8 +149,8 @@ Sample output excerpt: ```bash optimize-anything optimize --no-seed \ --objective "Draft a concise support policy prompt" \ - --model openai/gpt-4o-mini \ - --judge-model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ + --judge-model openai/gpt-5.6-luna \ --budget 25 ``` @@ -161,7 +163,7 @@ optimize-anything optimize --no-seed \ ```bash optimize-anything optimize strategy.md \ --evaluator-command bash eval_unbounded.sh \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --objective "Maximize reward" \ --score-range any ``` diff --git a/PROTOCOL.md b/PROTOCOL.md index 143a944..a694f65 100644 --- a/PROTOCOL.md +++ b/PROTOCOL.md @@ -27,7 +27,7 @@ Protocol v2 extends the input payload with optional metadata and example context { "_protocol_version": 2, "candidate": "", - "task_model": "openai/gpt-4o-mini", + "task_model": "openai/gpt-5.6-luna", "example": { "...": "dataset example object" } diff --git a/README.md b/README.md index 82d8187..537e4bc 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,9 @@ optimize-anything generate-evaluator seed.txt \ # 4) Optimize optimize-anything optimize seed.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Improve clarity and specificity" \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 20 \ --workers 4 \ --cache \ @@ -30,6 +30,10 @@ optimize-anything optimize seed.txt \ CLI stdout returns a JSON summary — see [Result Contract](#result-contract) for the full shape. +### Model defaults + +The proposer defaults to `openai/gpt-5.6-sol` after checking `--model` and `OPTIMIZE_ANYTHING_MODEL`. Generated judge and composite evaluators default to `openai/gpt-5.6-luna`. LLM judge calls use each provider's sampling defaults unless an explicit temperature is supplied. + ## How It Works optimize-anything runs a GEPA (Guided Evolutionary Prompt Algorithm) loop: propose → evaluate → reflect, repeating until budget is exhausted or early stopping kicks in. @@ -55,11 +59,11 @@ Use `--dataset` for multi-task optimization (one evaluator call per example). Ad ```bash optimize-anything optimize prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Generalize across customer request types" \ --dataset data/train.jsonl \ --valset data/val.jsonl \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 120 --workers 6 --cache --run-dir runs ``` @@ -69,11 +73,13 @@ Cross-check one artifact with multiple judge providers: ```bash optimize-anything validate result.txt \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 google/gemini-2.0-flash \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score clarity, constraints, and robustness" \ --intake-file intake.json ``` +Gemini uses LiteLLM's `gemini/` provider prefix. + ### Seedless mode No seed file required; GEPA bootstraps from objective. @@ -81,8 +87,8 @@ No seed file required; GEPA bootstraps from objective. ```bash optimize-anything optimize --no-seed \ --objective "Draft a concise, testable API prompt" \ - --model openai/gpt-4o-mini \ - --judge-model openai/gpt-4o-mini + --model openai/gpt-5.6-sol \ + --judge-model openai/gpt-5.6-luna ``` `--no-seed` requires both `--objective` and `--model`. @@ -110,7 +116,7 @@ to copying the prior fitness cache. ```bash optimize-anything optimize seed.txt \ --evaluator-command bash eval.sh \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 150 \ --cache --cache-from runs/run-20260303-120000 \ --run-dir runs \ @@ -126,7 +132,7 @@ For command/HTTP evaluators: ```bash optimize-anything optimize seed.txt \ --evaluator-command bash eval.sh \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --score-range any ``` @@ -226,7 +232,7 @@ The plugin includes three skills that Claude Code can invoke automatically: /optimize-anything:quick prompt.txt "improve clarity and specificity" → runs analyze + optimize with sensible defaults, shows diff -/optimize-anything:validate result.txt --providers openai/gpt-4o anthropic/claude-sonnet-4-5 +/optimize-anything:validate result.txt --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash → cross-checks the result with multiple judges ``` @@ -278,12 +284,12 @@ Pass it inline or from a file: # Inline optimize-anything optimize seed.txt \ --intake-json '{"quality_dimensions": ["clarity", "specificity"], "hard_constraints": ["max 100 words"]}' \ - --judge-model openai/gpt-4o-mini + --judge-model openai/gpt-5.6-luna # From file optimize-anything optimize seed.txt \ --intake-file intake.json \ - --judge-model openai/gpt-4o-mini + --judge-model openai/gpt-5.6-luna ``` `optimize-anything intake` normalizes and validates these keys: diff --git a/WALKTHROUGH.md b/WALKTHROUGH.md index 34feb5d..395e01a 100644 --- a/WALKTHROUGH.md +++ b/WALKTHROUGH.md @@ -57,7 +57,7 @@ echo '{"_protocol_version":2,"candidate":"test"}' | python evaluators/eval.py ```bash uv run optimize-anything score seed.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Score clarity and constraints" ``` @@ -65,9 +65,9 @@ uv run optimize-anything score seed.txt \ ```bash uv run optimize-anything optimize seed.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Improve clarity and specificity" \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 40 \ --cache --run-dir runs --diff ``` @@ -80,7 +80,7 @@ Notes: ```bash uv run optimize-anything score runs/run-/best_artifact.txt \ - --judge-model anthropic/claude-sonnet-4-5 \ + --judge-model anthropic/claude-sonnet-5 \ --objective "Score clarity, constraints, and usefulness" ``` @@ -88,20 +88,22 @@ uv run optimize-anything score runs/run-/best_artifact.txt \ ```bash uv run optimize-anything validate runs/run-/best_artifact.txt \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 google/gemini-2.0-flash \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score clarity, constraints, and robustness" ``` +Gemini uses LiteLLM's `gemini/` provider prefix. + ## Step 9: Iterate (dataset/generalization optional) For multi-task optimization: ```bash uv run optimize-anything optimize seed.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Generalize across support scenarios" \ --dataset data/train.jsonl --valset data/val.jsonl \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 120 --cache --run-dir runs ``` @@ -114,7 +116,7 @@ evaluators, cap concurrency with workers: uv run optimize-anything optimize seed.txt \ --evaluator-command bash evaluators/eval.sh \ --objective "Improve quality" \ - --model openai/gpt-4o-mini \ + --model openai/gpt-5.6-sol \ --budget 100 \ --workers 8 ``` diff --git a/commands/analyze.md b/commands/analyze.md index 5d66cb6..19674ed 100644 --- a/commands/analyze.md +++ b/commands/analyze.md @@ -10,14 +10,14 @@ Use an LLM to discover relevant quality dimensions for a given artifact and opti ## Usage ```bash -optimize-anything analyze SEED_FILE --judge-model openai/gpt-4o-mini --objective "Quality" +optimize-anything analyze SEED_FILE --judge-model openai/gpt-5.6-luna --objective "Quality" ``` ## Example ```bash optimize-anything analyze my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Score for clarity and persuasiveness" ``` @@ -26,7 +26,7 @@ After dimension discovery, proceed directly to optimization using the returned ` ```bash optimize-anything optimize my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Score for clarity and persuasiveness" \ --intake-json '' \ --budget 50 --diff --run-dir runs/ --early-stop diff --git a/commands/quick.md b/commands/quick.md index 5cf538c..bd9ac57 100644 --- a/commands/quick.md +++ b/commands/quick.md @@ -9,10 +9,10 @@ Run a no-questions-asked fast optimization. ## Behavior (do not ask follow-up questions) 1. Run analysis to discover dimensions: - - `optimize-anything analyze --judge-model openai/gpt-4o-mini --objective ""` + - `optimize-anything analyze --judge-model openai/gpt-5.6-luna --objective ""` - If analyze fails, skip dimension discovery and run optimize with `--judge-model` directly using the objective as-is. 2. Run optimization using LLM judge with: - - `--judge-model openai/gpt-4o-mini` + - `--judge-model openai/gpt-5.6-luna` - `--budget 50` - `--diff` - `--early-stop` diff --git a/commands/score.md b/commands/score.md index d8dee91..bae0e14 100644 --- a/commands/score.md +++ b/commands/score.md @@ -11,7 +11,7 @@ Score a single artifact file using a command evaluator, HTTP evaluator, or LLM j ``` optimize-anything score SEED_FILE --evaluator-command bash eval.sh -optimize-anything score SEED_FILE --judge-model openai/gpt-4o-mini --objective "Score clarity" +optimize-anything score SEED_FILE --judge-model openai/gpt-5.6-luna --objective "Score clarity" ``` ## Example @@ -21,7 +21,7 @@ optimize-anything score my-prompt.txt \ --evaluator-command bash evaluators/clarity.sh optimize-anything score my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Score for persuasiveness" ``` diff --git a/commands/validate.md b/commands/validate.md index 90812a0..d09dd37 100644 --- a/commands/validate.md +++ b/commands/validate.md @@ -12,10 +12,12 @@ Use multiple LLM judges to verify that a quality improvement is not provider-spe ## Usage ```bash optimize-anything validate \ - --providers openai/gpt-4o-mini anthropic/claude-sonnet-4-5 \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --objective "Score for clarity and constraint adherence" ``` +Gemini uses LiteLLM's `gemini/` provider prefix. + Optional: - `--intake-json` or `--intake-file` for shared dimensions/constraints - `--api-base` for custom provider endpoints diff --git a/evaluator-cookbook.md b/evaluator-cookbook.md index 2a84697..8d2bdc3 100644 --- a/evaluator-cookbook.md +++ b/evaluator-cookbook.md @@ -15,7 +15,7 @@ Your evaluator reads a JSON object from **stdin** (command) or as a **POST body* "_protocol_version": 2, "candidate": "the text being optimized", "example": {"input": "...", "expected": "..."}, - "task_model": "openai/gpt-4o-mini" + "task_model": "openai/gpt-5.6-luna" } ``` @@ -39,7 +39,7 @@ The `example` field is populated **per-call** when using `--dataset` mode. Each For shell-command evaluators, `task_model` is also available as an environment variable: ```bash -echo "$OPTIMIZE_ANYTHING_TASK_MODEL" # e.g. openai/gpt-4o-mini +echo "$OPTIMIZE_ANYTHING_TASK_MODEL" # e.g. openai/gpt-5.6-luna ``` ### Output @@ -183,14 +183,14 @@ Use an LLM judge when you want to score text qualitatively (clarity, style, pers **Score a single artifact:** ```bash uv run optimize-anything score my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Score clarity and persuasiveness" ``` **Optimize with an LLM judge:** ```bash uv run optimize-anything optimize my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Maximize clarity and persuasiveness" \ --budget 15 ``` @@ -198,7 +198,7 @@ uv run optimize-anything optimize my-prompt.txt \ **Discover quality dimensions, then optimize:** ```bash uv run optimize-anything analyze my-prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Quality" ``` @@ -206,7 +206,7 @@ Outcome: The `analyze` command returns named dimensions (e.g., clarity, specific ```bash uv run optimize-anything optimize prompt.txt \ - --judge-model openai/gpt-4o-mini \ + --judge-model openai/gpt-5.6-luna \ --objective "Maximize quality" \ --intake-json '{ "quality_dimensions": [ @@ -254,12 +254,12 @@ def main(): print(json.dumps({"score": 0.0, "error": "example missing input field"})) return 0 - model = os.getenv("JUDGE_MODEL", "openai/gpt-4o-mini") + model = os.getenv("JUDGE_MODEL", "openai/gpt-5.6-luna") try: resp = completion(model=model, messages=[ {"role": "system", "content": candidate}, {"role": "user", "content": user_input}, - ], temperature=0, max_tokens=256) + ], max_tokens=256) output = (resp.choices[0].message.content or "").lower() except Exception as exc: print(json.dumps({"score": 0.0, "error": f"llm_call_failed: {exc}"})) @@ -337,7 +337,7 @@ def llm_judge(candidate, model): resp = completion(model=model, messages=[ {"role": "system", "content": "Return JSON: {score, clarity, completeness, reasoning} all 0-1."}, {"role": "user", "content": f"Rate clarity and completeness:\n\n{candidate}"}, - ], temperature=0, response_format={"type": "json_object"}) + ], response_format={"type": "json_object"}) p = json.loads(resp.choices[0].message.content or "{}") return clamp01(float(p.get("score", 0))), p except Exception as e: @@ -346,7 +346,7 @@ def llm_judge(candidate, model): def main(): payload = json.load(sys.stdin) candidate = str(payload.get("candidate", "")) - model = os.getenv("JUDGE_MODEL", "openai/gpt-4o-mini") + model = os.getenv("JUDGE_MODEL", "openai/gpt-5.6-luna") h_score, h_side = heuristic_score(candidate.strip()) if h_score < 0.2: # short-circuit: skip LLM @@ -462,7 +462,7 @@ else: ```bash uv run optimize-anything optimize prompt.txt \ --evaluator-command python3 eval.py \ - --task-model openai/gpt-4o-mini \ + --task-model openai/gpt-5.6-luna \ --objective "Optimize for the target model" \ --budget 15 ``` diff --git a/scripts/live_integration.py b/scripts/live_integration.py index 7938822..a767156 100644 --- a/scripts/live_integration.py +++ b/scripts/live_integration.py @@ -5,18 +5,19 @@ structured JSON. Designed to be driven by Claude Code as an interactive observer. Usage: - # GREEN phase: optimize an artifact + # GREEN phase: optimize with the proposer-tier default python scripts/live_integration.py --phase green \ --artifact skills/generate-evaluator/SKILL.md \ - --evaluator-command bash evaluators/skill_clarity.sh \ --budget 15 --objective "Improve clarity" \ - --run-dir integration_runs + --run-dir integration_runs \ + --model openai/gpt-5.6-sol \ + --evaluator-command bash evaluators/skill_clarity.sh - # RED phase: score with multiple providers + # RED phase: validate with the canonical judge-tier provider set python scripts/live_integration.py --phase red \ --artifact skills/generate-evaluator/SKILL.md \ --objective "Score skill quality" \ - --providers openai/gpt-5.1 anthropic/claude-sonnet-4-5-20250929 \ + --providers openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash \ --evaluator-command bash evaluators/skill_clarity.sh """ from __future__ import annotations @@ -45,16 +46,16 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--run-dir", help="GREEN: directory to save run artifacts") parser.add_argument( "--model", - help="GREEN: LLM model string for the proposer (e.g. 'openai/gpt-4o-mini')", + help="GREEN proposer model (e.g. 'openai/gpt-5.6-sol')", ) parser.add_argument( "--providers", nargs="+", - help="RED: LLM provider model strings for multi-provider scoring", + help="RED validation models (e.g. openai/gpt-5.6-luna anthropic/claude-sonnet-5 gemini/gemini-3.6-flash)", ) parser.add_argument( "--judge-model", - help="GREEN: LLM judge model for meta-evaluator optimization", + help="GREEN judge model (e.g. 'openai/gpt-5.6-luna')", ) parser.add_argument( "--judge-objective", @@ -261,7 +262,7 @@ def _run_red(args: argparse.Namespace) -> int: providers = args.providers or [] for provider in providers: judge_score = _score_with_judge(artifact_path, provider, args.objective) - # Use a clean key: "openai/gpt-5.1" -> "openai_gpt_5_1" + # Use a clean key: "openai/gpt-5.6-luna" -> "openai_gpt_5_6_luna" key = provider.replace("/", "_").replace("-", "_").replace(".", "_") scores[key] = judge_score diff --git a/scripts/plugin_eval.sh b/scripts/plugin_eval.sh index c93887b..09bbb45 100755 --- a/scripts/plugin_eval.sh +++ b/scripts/plugin_eval.sh @@ -81,17 +81,17 @@ scenario_budget() { scenario_analyze() { run_scenario "analyze" \ - "Use the optimize-anything plugin to analyze the artifact at $SEED_FILE for quality dimensions. Run: optimize-anything analyze $SEED_FILE --judge-model openai/gpt-4o-mini --objective 'Score the quality of this system prompt'" + "Use the optimize-anything plugin to analyze the artifact at $SEED_FILE for quality dimensions. Run: optimize-anything analyze $SEED_FILE --judge-model openai/gpt-5.6-luna --objective 'Score the quality of this system prompt'" } scenario_validate() { run_scenario "validate" \ - "Use the optimize-anything plugin to validate the artifact at $SEED_FILE across multiple providers. Run: optimize-anything validate $SEED_FILE --providers openai/gpt-4o-mini anthropic/claude-haiku-4-5-20251001 --objective 'Score the quality and clarity of this system prompt'" + "Use the optimize-anything plugin to validate the artifact at $SEED_FILE across multiple providers. Run: optimize-anything validate $SEED_FILE --providers openai/gpt-5.6-luna anthropic/claude-haiku-4-5-20251001 --objective 'Score the quality and clarity of this system prompt'" } scenario_quick() { run_scenario "quick" \ - "Use the optimize-anything plugin to quickly optimize the seed at $SEED_FILE. Run: optimize-anything optimize $SEED_FILE --judge-model openai/gpt-4o-mini --objective 'Improve clarity and specificity of this system prompt' --budget 5 --model openai/gpt-4o-mini --output $RESULTS_DIR/quick-best.txt" + "Use the optimize-anything plugin to quickly optimize the seed at $SEED_FILE. Run: optimize-anything optimize $SEED_FILE --judge-model openai/gpt-5.6-luna --objective 'Improve clarity and specificity of this system prompt' --budget 5 --model openai/gpt-5.6-sol --output $RESULTS_DIR/quick-best.txt" } case "$SCENARIO" in diff --git a/scripts/plugin_regression.py b/scripts/plugin_regression.py index ef46a14..2192317 100644 --- a/scripts/plugin_regression.py +++ b/scripts/plugin_regression.py @@ -104,7 +104,7 @@ def _assert_contains(result: str, scenario: str, needles: list[str]) -> None: def scenario_analyze(repo_root: Path, output_dir: Path, seed_path: Path) -> dict[str, Any]: prompt = ( f"Use the optimize-anything plugin to analyze the artifact at {seed_path} for quality dimensions. " - "Run: optimize-anything analyze runs/zo-eval/seed.txt --judge-model openai/gpt-4o-mini " + "Run: optimize-anything analyze runs/zo-eval/seed.txt --judge-model openai/gpt-5.6-luna " "--objective 'Score the quality of this system prompt'" ) payload = _run_claude(repo_root, prompt, output_dir / "analyze.json", output_dir / "analyze.stderr.log") @@ -121,7 +121,7 @@ def scenario_analyze(repo_root: Path, output_dir: Path, seed_path: Path) -> dict def scenario_validate(repo_root: Path, output_dir: Path, seed_path: Path) -> dict[str, Any]: prompt = ( f"Use the optimize-anything plugin to validate the artifact at {seed_path} across multiple providers. " - "Run: optimize-anything validate runs/zo-eval/seed.txt --providers openai/gpt-4o-mini " + "Run: optimize-anything validate runs/zo-eval/seed.txt --providers openai/gpt-5.6-luna " "anthropic/claude-haiku-4-5-20251001 --objective 'Score the quality and clarity of this system prompt'" ) payload = _run_claude(repo_root, prompt, output_dir / "validate.json", output_dir / "validate.stderr.log") @@ -139,9 +139,9 @@ def scenario_quick(repo_root: Path, output_dir: Path, seed_path: Path) -> dict[s best_path = repo_root / "runs" / "plugin-eval" / "quick-best.txt" prompt = ( f"Use the optimize-anything plugin to quickly optimize the seed at {seed_path}. " - f"Run: optimize-anything optimize runs/zo-eval/seed.txt --judge-model openai/gpt-4o-mini " + f"Run: optimize-anything optimize runs/zo-eval/seed.txt --judge-model openai/gpt-5.6-luna " f"--objective 'Improve clarity and specificity of this system prompt' --budget 5 " - f"--model openai/gpt-4o-mini --output {best_path}" + f"--model openai/gpt-5.6-sol --output {best_path}" ) payload = _run_claude(repo_root, prompt, output_dir / "quick.json", output_dir / "quick.stderr.log") result = _assert_success(payload, "quick") diff --git a/skills/evaluator-patterns/SKILL.md b/skills/evaluator-patterns/SKILL.md index edc57eb..c1b0e58 100644 --- a/skills/evaluator-patterns/SKILL.md +++ b/skills/evaluator-patterns/SKILL.md @@ -82,7 +82,7 @@ Return JSON exactly like: }} """.strip() - model = os.getenv("JUDGE_MODEL", "openai/gpt-4o-mini") + model = os.getenv("JUDGE_MODEL", "openai/gpt-5.6-luna") try: resp = completion( @@ -91,7 +91,6 @@ Return JSON exactly like: {"role": "system", "content": system_prompt}, {"role": "user", "content": user_prompt}, ], - temperature=0, response_format={"type": "json_object"}, ) content = resp.choices[0].message.content diff --git a/skills/generate-evaluator/SKILL.md b/skills/generate-evaluator/SKILL.md index 7ec2e1f..ecd9f10 100644 --- a/skills/generate-evaluator/SKILL.md +++ b/skills/generate-evaluator/SKILL.md @@ -61,7 +61,7 @@ Generate a judge evaluator and test it: # Generate optimize-anything generate-evaluator seed.txt \ --objective "Score clarity and specificity" \ - --model openai/gpt-4o-mini > eval_judge.py + --model openai/gpt-5.6-luna > eval_judge.py # Test it echo '{"candidate":"Your artifact text here"}' | python3 eval_judge.py diff --git a/src/optimize_anything/cli.py b/src/optimize_anything/cli.py index 71eb44c..905fe05 100644 --- a/src/optimize_anything/cli.py +++ b/src/optimize_anything/cli.py @@ -14,6 +14,7 @@ _preflight_command_evaluator, _preflight_http_evaluator, ) +from optimize_anything.model_defaults import DEFAULT_EVALUATOR_MODEL EvaluatorFn = Callable[..., tuple[float, dict[str, Any]]] EvaluatorFactory = Callable[..., EvaluatorFn] @@ -60,16 +61,16 @@ def main(argv: list[str] | None = None) -> int: opt_parser.add_argument( "--model", help=( - "LiteLLM model string for the proposer LLM " - "(e.g. 'openai/gpt-4o-mini', 'claude-sonnet-4-6'). " + "LiteLLM model string for the proposer LLM. Proposer: " + "openai/gpt-5.6-sol or anthropic/claude-sonnet-5. " "Falls back to OPTIMIZE_ANYTHING_MODEL env var." ), ) opt_parser.add_argument( "--judge-model", help=( - "LiteLLM model string for built-in LLM-as-judge evaluation " - "(e.g. 'openai/gpt-4o-mini'). " + "LiteLLM model string for built-in LLM-as-judge evaluation. " + "Judge: openai/gpt-5.6-luna. " "Mutually exclusive with --evaluator-command and --evaluator-url." ), ) @@ -187,8 +188,11 @@ def main(argv: list[str] | None = None) -> int: ) gen_parser.add_argument( "--model", - default="openai/gpt-4o-mini", - help="LiteLLM model to hardcode in generated judge/composite evaluators", + default=DEFAULT_EVALUATOR_MODEL, + help=( + "LiteLLM model to hardcode in generated judge/composite evaluators " + f"(default: {DEFAULT_EVALUATOR_MODEL})" + ), ) gen_parser.add_argument( "--dataset", @@ -254,8 +258,8 @@ def main(argv: list[str] | None = None) -> int: score_parser.add_argument( "--judge-model", help=( - "LiteLLM model string for LLM-as-judge scoring " - "(e.g. 'openai/gpt-5.1'). " + "LiteLLM model string for LLM-as-judge scoring. " + "Judge: openai/gpt-5.6-luna. " "Mutually exclusive with --evaluator-command and --evaluator-url." ), ) @@ -298,7 +302,11 @@ def main(argv: list[str] | None = None) -> int: "--providers", nargs="+", required=True, - help="Two or more LiteLLM provider model strings (e.g. openai/gpt-4o-mini anthropic/claude-sonnet-4-5)", + help=( + "Two or more LiteLLM provider model strings. Validation: " + "openai/gpt-5.6-luna anthropic/claude-sonnet-5 " + "gemini/gemini-3.6-flash" + ), ) validate_parser.add_argument( "--objective", @@ -327,7 +335,7 @@ def main(argv: list[str] | None = None) -> int: analyze_parser.add_argument( "--judge-model", required=True, - help="LiteLLM model string for the LLM judge (e.g. 'openai/gpt-4o-mini')", + help="LiteLLM model string for the LLM judge. Judge: openai/gpt-5.6-luna.", ) analyze_parser.add_argument( "--objective", @@ -341,8 +349,11 @@ def main(argv: list[str] | None = None) -> int: analyze_parser.add_argument( "--temperature", type=float, - default=0.0, - help="Sampling temperature for LLM calls (default: 0.0)", + default=None, + help=( + "Sampling temperature for LLM calls " + "(default: provider model default)" + ), ) analyze_parser.add_argument( "--timeout", diff --git a/src/optimize_anything/cli_optimize.py b/src/optimize_anything/cli_optimize.py index 5860d94..05140cc 100644 --- a/src/optimize_anything/cli_optimize.py +++ b/src/optimize_anything/cli_optimize.py @@ -5,7 +5,6 @@ import argparse import copy import json -import os import sys from pathlib import Path from typing import Any @@ -17,6 +16,7 @@ _save_run_dir, _timestamped_run_dir, ) +from optimize_anything.model_defaults import resolve_proposer_model OptimizeInputs = tuple[ argparse.Namespace, @@ -67,7 +67,7 @@ def _cmd_optimize(args: argparse.Namespace) -> int: file=sys.stderr, ) - model = args.model or os.environ.get("OPTIMIZE_ANYTHING_MODEL") + model = resolve_proposer_model(args.model) config, gepa_run_dir, early_stop_active, runtime_error = _build_optimize_runtime( args, model=model, @@ -134,7 +134,7 @@ def _cmd_optimize(args: argparse.Namespace) -> int: def _build_optimize_runtime( args: argparse.Namespace, *, - model: str | None, + model: str, ) -> tuple[Any, str | None, bool, str | None]: """Build GEPA runtime config plus run-dir state for optimize.""" from optimize_anything.stop import plateau_stop_callback @@ -184,14 +184,11 @@ def _build_optimize_runtime( n=args.proposals_per_iteration ) engine = EngineConfig(**engine_kwargs) - if model: - config = GEPAConfig( - engine=engine, - reflection=ReflectionConfig(reflection_lm=model), - stop_callbacks=stop_callbacks, - ) - else: - config = GEPAConfig(engine=engine, stop_callbacks=stop_callbacks) + config = GEPAConfig( + engine=engine, + reflection=ReflectionConfig(reflection_lm=model), + stop_callbacks=stop_callbacks, + ) return config, gepa_run_dir, early_stop_active, None diff --git a/src/optimize_anything/evaluator_generator.py b/src/optimize_anything/evaluator_generator.py index 8aa5464..ce5207b 100644 --- a/src/optimize_anything/evaluator_generator.py +++ b/src/optimize_anything/evaluator_generator.py @@ -6,6 +6,8 @@ from numbers import Real from typing import Any +from optimize_anything.model_defaults import DEFAULT_EVALUATOR_MODEL + def generate_evaluator_script( *, @@ -13,7 +15,7 @@ def generate_evaluator_script( objective: str, evaluator_type: str | None = None, intake: Mapping[str, Any] | None = None, - model: str = "openai/gpt-4o-mini", + model: str = DEFAULT_EVALUATOR_MODEL, dataset: bool = False, ) -> str: """Generate an evaluator script that reads input JSON and outputs score JSON.""" @@ -414,7 +416,7 @@ def _generate_judge_evaluator( template_family: str, rubric_summary: str, quality_dimensions: list[tuple[str, float]], - model: str = "openai/gpt-4o-mini", + model: str, dataset: bool = False, ) -> str: """Generate a Python LLM-judge evaluator script using litellm.""" @@ -437,7 +439,7 @@ def _generate_judge_evaluator( def _build_prompt(candidate: str, example: object | None) -> str: dimensions_text = "\\n".join([f"- {{name}} (weight={{weight}})" for name, weight in QUALITY_DIMENSIONS]) example_text = json.dumps(example, ensure_ascii=False, indent=2) if example is not None else "(none)" - return f\"\"\"## Objective\n{{OBJECTIVE}}\n\n## Template Family\n{{TEMPLATE_FAMILY}}\n\n## Rubric Summary\n{{RUBRIC_SUMMARY}}\n\n## Quality Dimensions\n{{dimensions_text}}\n\n## Example Context (optional)\n{{example_text}}\n\n## Artifact to Evaluate\n```\n{{candidate}}\n```\n\nReturn JSON with keys: score, reasoning, and one key per quality dimension name. score must be in [0,1].\"\"\" + return f\"\"\"## Objective\\n{{OBJECTIVE}}\\n\\n## Template Family\\n{{TEMPLATE_FAMILY}}\\n\\n## Rubric Summary\\n{{RUBRIC_SUMMARY}}\\n\\n## Quality Dimensions\\n{{dimensions_text}}\\n\\n## Example Context (optional)\\n{{example_text}}\\n\\n## Artifact to Evaluate\\n```\\n{{candidate}}\\n```\\n\\nReturn JSON with keys: score, reasoning, and one key per quality dimension name. score must be in [0,1].\"\"\" def _api_key_available() -> bool: key_vars = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", "OPENROUTER_API_KEY"] @@ -446,7 +448,7 @@ def _api_key_available() -> bool: def _strip_code_fences(text: str) -> str: cleaned = text.strip() if cleaned.startswith("```"): - first_newline = cleaned.index("\n") if "\n" in cleaned else len(cleaned) + first_newline = cleaned.index("\\n") if "\\n" in cleaned else len(cleaned) cleaned = cleaned[first_newline + 1:] if cleaned.rstrip().endswith("```"): cleaned = cleaned.rstrip()[:-len("```")].rstrip() @@ -478,7 +480,6 @@ def main() -> int: {{"role": "system", "content": JUDGE_SYSTEM_PROMPT}}, {{"role": "user", "content": prompt}}, ], - temperature=0.0, timeout=60.0, response_format={{"type": "json_object"}}, ) @@ -524,7 +525,7 @@ def _generate_composite_evaluator( template_family: str, rubric_summary: str, quality_dimensions: list[tuple[str, float]], - model: str = "openai/gpt-4o-mini", + model: str, dataset: bool = False, ) -> str: """Generate composite evaluator with hard constraints + judge scoring.""" @@ -544,6 +545,8 @@ def _generate_composite_evaluator( import sys # Composite evaluator: hard constraints first, then LLM judge. + MODEL = {model!r} + def _constraint_non_empty(candidate: str) -> tuple[bool, str]: if candidate.strip(): return True, "" @@ -564,7 +567,7 @@ def _constraint_no_placeholder(candidate: str) -> tuple[bool, str]: def _strip_code_fences(text: str) -> str: cleaned = text.strip() if cleaned.startswith("```"): - first_newline = cleaned.index("\n") if "\n" in cleaned else len(cleaned) + first_newline = cleaned.index("\\n") if "\\n" in cleaned else len(cleaned) cleaned = cleaned[first_newline + 1:] if cleaned.rstrip().endswith("```"): cleaned = cleaned.rstrip()[:-len("```")].rstrip() diff --git a/src/optimize_anything/llm_judge.py b/src/optimize_anything/llm_judge.py index 99d72d6..23b4168 100644 --- a/src/optimize_anything/llm_judge.py +++ b/src/optimize_anything/llm_judge.py @@ -85,7 +85,7 @@ def llm_judge_evaluator( quality_dimensions: list[dict[str, Any]] | None = None, hard_constraints: list[str] | None = None, timeout: float = 60.0, - temperature: float = 0.0, + temperature: float | None = None, api_base: str | None = None, task_model: str | None = None, ) -> Callable[[str, Any | None], tuple[float, dict[str, Any]]]: @@ -113,10 +113,11 @@ def evaluate(candidate: str, example: Any | None = None) -> tuple[float, dict[st {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, {"role": "user", "content": prompt}, ], - "temperature": temperature, "timeout": timeout, "response_format": {"type": "json_object"}, } + if temperature is not None: + completion_kwargs["temperature"] = temperature if api_base: completion_kwargs["base_url"] = api_base @@ -321,7 +322,7 @@ def analyze_for_dimensions( *, api_base: str | None = None, timeout: float = 60.0, - temperature: float = 0.0, + temperature: float | None = None, ) -> dict[str, Any]: """Score an artifact then discover quality dimensions for refinement. @@ -346,10 +347,11 @@ def analyze_for_dimensions( {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, {"role": "user", "content": score_prompt}, ], - "temperature": temperature, "timeout": timeout, "response_format": {"type": "json_object"}, } + if temperature is not None: + completion_kwargs["temperature"] = temperature if api_base: completion_kwargs["base_url"] = api_base @@ -375,10 +377,11 @@ def analyze_for_dimensions( {"role": "system", "content": ANALYZE_SYSTEM_PROMPT}, {"role": "user", "content": analyze_prompt}, ], - "temperature": temperature, "timeout": timeout, "response_format": {"type": "json_object"}, } + if temperature is not None: + analyze_kwargs["temperature"] = temperature if api_base: analyze_kwargs["base_url"] = api_base diff --git a/src/optimize_anything/model_defaults.py b/src/optimize_anything/model_defaults.py new file mode 100644 index 0000000..d9cffae --- /dev/null +++ b/src/optimize_anything/model_defaults.py @@ -0,0 +1,23 @@ +"""Project-owned defaults for LLM roles.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from typing import Final + +DEFAULT_PROPOSER_MODEL: Final[str] = "openai/gpt-5.6-sol" +DEFAULT_EVALUATOR_MODEL: Final[str] = "openai/gpt-5.6-luna" + + +def resolve_proposer_model( + explicit_model: str | None, + environ: Mapping[str, str] | None = None, +) -> str: + """Resolve CLI, environment, then project default proposer precedence.""" + environment = os.environ if environ is None else environ + return ( + explicit_model + or environment.get("OPTIMIZE_ANYTHING_MODEL") + or DEFAULT_PROPOSER_MODEL + ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9e0eb93..ec8c6ca 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,6 +23,16 @@ def test_optimize_help(self, capsys): assert "--intake-json" in captured.out assert "--intake-file" in captured.out assert "--evaluator-cwd" in captured.out + assert "openai/gpt-5.6-sol" in captured.out + + def test_generate_evaluator_help(self, capsys): + """Verify generated evaluator help recommends the current judge model.""" + try: + main(["generate-evaluator", "--help"]) + except SystemExit as e: + assert e.code == 0 + captured = capsys.readouterr() + assert "openai/gpt-5.6-luna" in captured.out def test_explain(self, tmp_path: Path, capsys): seed_file = tmp_path / "seed.txt" @@ -856,6 +866,44 @@ def fake_optimize(**kwargs): cfg = captured_config["config"] assert cfg.reflection.reflection_lm == "gemini/gemini-2.0-flash" + def test_optimize_uses_default_proposer_model( + self, tmp_path: Path, capsys, monkeypatch + ): + seed_file = tmp_path / "seed.txt" + seed_file.write_text("test") + captured_config = {} + + class DummyResult: + best_candidate = "x" + total_metric_calls = 1 + + def fake_optimize(**kwargs): + captured_config["config"] = kwargs.get("config") + return DummyResult() + + monkeypatch.setattr( + "optimize_anything.evaluators.command_evaluator", + lambda command, cwd=None, **kwargs: lambda c: (0.5, {}), + ) + monkeypatch.setattr( + "gepa.optimize_anything.optimize_anything", + fake_optimize, + ) + monkeypatch.setattr( + "optimize_anything.cli._preflight_command_evaluator", + lambda command, cwd=None: None, + ) + monkeypatch.delenv("OPTIMIZE_ANYTHING_MODEL", raising=False) + + result = main([ + "optimize", str(seed_file), + "--evaluator-command", "bash", "eval.sh", + "--budget", "1", + ]) + assert result == 0 + cfg = captured_config["config"] + assert cfg.reflection.reflection_lm == "openai/gpt-5.6-sol" + def test_optimize_prints_progress_to_stderr( self, tmp_path: Path, capsys, monkeypatch ): diff --git a/tests/test_evaluator_generator.py b/tests/test_evaluator_generator.py index 82c1687..3324684 100644 --- a/tests/test_evaluator_generator.py +++ b/tests/test_evaluator_generator.py @@ -1,11 +1,60 @@ """Tests for evaluator generator.""" import json import subprocess +import sys from pathlib import Path +import pytest + from optimize_anything.evaluator_generator import generate_evaluator_script +@pytest.mark.parametrize("evaluator_type", ["judge", "composite"]) +def test_generated_python_evaluator_compiles(evaluator_type: str) -> None: + script = generate_evaluator_script( + seed="hello", + objective="score quality", + evaluator_type=evaluator_type, + ) + + compile(script, f"", "exec") + + +@pytest.mark.parametrize("evaluator_type", ["judge", "composite"]) +def test_generated_python_evaluator_reports_missing_key_offline( + evaluator_type: str, + tmp_path: Path, +) -> None: + script = generate_evaluator_script( + seed="hello", + objective="score quality", + evaluator_type=evaluator_type, + ) + script_path = tmp_path / f"{evaluator_type}_evaluator.py" + script_path.write_text(script, encoding="utf-8") + + proc = subprocess.run( + [sys.executable, str(script_path)], + input=json.dumps({"candidate": "hello"}), + capture_output=True, + text=True, + check=True, + env={}, + ) + + result = json.loads(proc.stdout) + assert result["error"] == "missing_api_key" + + +def test_generated_judge_uses_provider_sampling_defaults() -> None: + script = generate_evaluator_script( + seed="hello", + objective="score quality", + evaluator_type="judge", + ) + assert "temperature=" not in script + + class TestGenerateEvaluatorScript: def test_command_evaluator_is_bash(self): script = generate_evaluator_script(seed="hello", objective="improve clarity", evaluator_type="command") @@ -39,6 +88,22 @@ def test_judge_evaluator_contains_litellm_and_objective(self): assert "from litellm import completion" in script assert objective in script + def test_default_judge_uses_current_evaluator_model(self) -> None: + script = generate_evaluator_script( + seed="hello", + objective="score quality", + evaluator_type="judge", + ) + assert "MODEL = 'openai/gpt-5.6-luna'" in script + + def test_default_composite_uses_current_evaluator_model(self) -> None: + script = generate_evaluator_script( + seed="hello", + objective="score quality", + evaluator_type="composite", + ) + assert "MODEL = 'openai/gpt-5.6-luna'" in script + def test_judge_evaluator_handles_missing_api_key_gracefully(self): script = generate_evaluator_script(seed="hello", objective="test", evaluator_type="judge") assert "Missing API key" in script diff --git a/tests/test_llm_judge.py b/tests/test_llm_judge.py index a4e569b..1e45605 100644 --- a/tests/test_llm_judge.py +++ b/tests/test_llm_judge.py @@ -164,6 +164,24 @@ def test_temperature_passed_through_to_litellm(self): call_kwargs = mock_call.call_args.kwargs assert call_kwargs["temperature"] == 0.7 + def test_default_temperature_is_omitted_from_litellm(self): + captured: dict[str, object] = {} + + def fake_completion(**kwargs): + captured.update(kwargs) + return self._make_mock_completion( + '{"score": 0.8, "reasoning": "Good"}' + ) + + with patch("litellm.completion", side_effect=fake_completion): + evaluate = llm_judge_evaluator( + "Score quality", + model="anthropic/claude-sonnet-5", + ) + evaluate("candidate") + + assert "temperature" not in captured + def test_model_string_passed_through(self): response_content = json.dumps({"score": 0.5, "reasoning": "ok"}) evaluator = llm_judge_evaluator( @@ -582,9 +600,12 @@ def test_makes_exactly_two_llm_calls(self): ] }) + captured_calls: list[dict[str, object]] = [] call_count = 0 + def mock_completion(**kwargs): nonlocal call_count + captured_calls.append(kwargs) call_count += 1 if call_count == 1: return self._make_mock_response(score_response) @@ -594,6 +615,31 @@ def mock_completion(**kwargs): analyze_for_dimensions("text", "obj", "openai/gpt-4o-mini") assert call_count == 2 + assert all("temperature" not in kwargs for kwargs in captured_calls) + + def test_explicit_temperature_is_forwarded_to_both_llm_calls(self): + score_response = json.dumps({"score": 0.5, "reasoning": "ok"}) + dims_response = json.dumps({ + "dimensions": [ + {"name": "a", "weight": 1.0, "score": 0.5, "description": "d"}, + ] + }) + responses = iter((score_response, dims_response)) + captured_calls: list[dict[str, object]] = [] + + def mock_completion(**kwargs): + captured_calls.append(kwargs) + return self._make_mock_response(next(responses)) + + with patch("litellm.completion", side_effect=mock_completion): + analyze_for_dimensions( + "text", + "obj", + "openai/gpt-4o-mini", + temperature=0.7, + ) + + assert [call["temperature"] for call in captured_calls] == [0.7, 0.7] def test_scoring_failure_raises_runtime_error(self): with patch("litellm.completion", side_effect=RuntimeError("API down")): diff --git a/tests/test_model_defaults.py b/tests/test_model_defaults.py new file mode 100644 index 0000000..148cd1e --- /dev/null +++ b/tests/test_model_defaults.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from optimize_anything.model_defaults import ( + DEFAULT_EVALUATOR_MODEL, + DEFAULT_PROPOSER_MODEL, + resolve_proposer_model, +) + + +ROOT = Path(__file__).resolve().parents[1] +EXECUTABLE_MODEL_REFERENCE_FILES = ( + "src/optimize_anything/cli.py", + "src/optimize_anything/evaluator_generator.py", + "scripts/live_integration.py", + "scripts/plugin_eval.sh", + "scripts/plugin_regression.py", + "skills/evaluator-patterns/SKILL.md", + "skills/generate-evaluator/SKILL.md", +) +DOCUMENTATION_MODEL_REFERENCE_FILES = ( + "AGENTS.md", + "CLAUDE.md", + "EXAMPLES.md", + "PROTOCOL.md", + "README.md", + "WALKTHROUGH.md", + "evaluator-cookbook.md", + "commands/analyze.md", + "commands/quick.md", + "commands/score.md", + "commands/validate.md", +) +STALE_CANONICAL_MODEL_STRINGS = ( + "openai/gpt-4o", + "openai/gpt-4o-mini", + "openai/gpt-5.1", + "claude-sonnet-4-6", + "anthropic/claude-sonnet-4-5", + "anthropic/claude-sonnet-4-5-20250929", + "anthropic/claude-sonnet-4-6", + "google/gemini-2.0-flash", + "gemini/gemini-2.0-flash", +) + + +@pytest.mark.parametrize( + "relative_path", + EXECUTABLE_MODEL_REFERENCE_FILES + DOCUMENTATION_MODEL_REFERENCE_FILES, +) +def test_user_facing_surfaces_do_not_recommend_stale_models( + relative_path: str, +) -> None: + text = (ROOT / relative_path).read_text(encoding="utf-8") + assert not [ + model for model in STALE_CANONICAL_MODEL_STRINGS if model in text + ] + + +def test_primary_docs_show_current_validation_models() -> None: + for relative_path in ("README.md", "EXAMPLES.md", "WALKTHROUGH.md"): + text = (ROOT / relative_path).read_text(encoding="utf-8") + assert "openai/gpt-5.6-luna" in text + assert "anthropic/claude-sonnet-5" in text + assert "gemini/gemini-3.6-flash" in text + + +def test_default_models_match_current_capability_tiers() -> None: + assert DEFAULT_PROPOSER_MODEL == "openai/gpt-5.6-sol" + assert DEFAULT_EVALUATOR_MODEL == "openai/gpt-5.6-luna" + + +@pytest.mark.parametrize( + ("explicit_model", "environ", "expected"), + [ + ( + "anthropic/claude-sonnet-5", + {"OPTIMIZE_ANYTHING_MODEL": "gemini/gemini-3.6-flash"}, + "anthropic/claude-sonnet-5", + ), + ( + None, + {"OPTIMIZE_ANYTHING_MODEL": "gemini/gemini-3.6-flash"}, + "gemini/gemini-3.6-flash", + ), + (None, {}, "openai/gpt-5.6-sol"), + ], +) +def test_resolve_proposer_model_precedence( + explicit_model: str | None, + environ: dict[str, str], + expected: str, +) -> None: + assert resolve_proposer_model(explicit_model, environ) == expected + + +@pytest.mark.parametrize( + ("model", "provider"), + [ + ("openai/gpt-5.6-sol", "openai"), + ("openai/gpt-5.6-luna", "openai"), + ("anthropic/claude-sonnet-5", "anthropic"), + ("gemini/gemini-3.6-flash", "gemini"), + ], +) +def test_recommended_models_route_through_pinned_litellm( + model: str, + provider: str, +) -> None: + import litellm + + assert litellm.get_llm_provider(model)[1] == provider