From 3f757f94ff9117c28f317298f451f4848f307cd2 Mon Sep 17 00:00:00 2001
From: Shuxin Lin <linshuhsin@gmail.com>
Date: Mon, 7 Sep 2026 04:06:29 +0000
Subject: [PATCH] Update benchmark run configuration

---
 benchmarks/run.sh                   | 63 +++++++++++++++++++++++------
 benchmarks/scenario_suite/lite.yaml | 48 +++++++++++++++++-----
 2 files changed, 87 insertions(+), 24 deletions(-)

diff --git a/benchmarks/run.sh b/benchmarks/run.sh
index 8ab880ff..27607f1e 100755
--- a/benchmarks/run.sh
+++ b/benchmarks/run.sh
@@ -2,27 +2,64 @@
 
 set -euo pipefail
 
-scenario_dir="${1:-${SCENARIO_DIR:-}}"
-leaderboard_dir="${2:-${LEADERBOARD_DIR:-}}"
+usage() {
+  printf 'Usage: %s -s SCENARIO_DIR -l LEADERBOARD_DIR -t TRAJECTORY_DIR\n' "$0" >&2
+  printf '       %s SCENARIO_DIR LEADERBOARD_DIR TRAJECTORY_DIR\n' "$0" >&2
+  printf '       or set SCENARIO_DIR, LEADERBOARD_DIR, and TRAJECTORY_DIR\n' >&2
+}
 
-if [[ -z "$scenario_dir" || -z "$leaderboard_dir" ]]; then
-  printf 'Usage: %s SCENARIO_DIR LEADERBOARD_DIR\n' "$0" >&2
-  printf '       or set SCENARIO_DIR and LEADERBOARD_DIR\n' >&2
+scenario_dir="${SCENARIO_DIR:-}"
+leaderboard_dir="${LEADERBOARD_DIR:-}"
+trajectory_dir="${TRAJECTORY_DIR:-}"
+
+while getopts ':s:l:t:' option; do
+  case "$option" in
+    s) scenario_dir="$OPTARG" ;;
+    l) leaderboard_dir="$OPTARG" ;;
+    t) trajectory_dir="$OPTARG" ;;
+    :) printf 'Option -%s requires an argument.\n' "$OPTARG" >&2; usage; exit 2 ;;
+    \?) printf 'Unknown option: -%s\n' "$OPTARG" >&2; usage; exit 2 ;;
+  esac
+done
+
+option_arg_count=$((OPTIND - 1))
+shift "$option_arg_count"
+
+if (( option_arg_count > 0 && $# > 0 )); then
+  printf 'Options and positional arguments cannot be combined.\n' >&2
+  usage
+  exit 2
+fi
+
+if (( option_arg_count == 0 )); then
+  if (( $# > 3 )); then
+    usage
+    exit 2
+  fi
+  scenario_dir="${1:-$scenario_dir}"
+  leaderboard_dir="${2:-$leaderboard_dir}"
+  trajectory_dir="${3:-$trajectory_dir}"
+fi
+
+if [[ -z "$scenario_dir" || -z "$leaderboard_dir" || -z "$trajectory_dir" ]]; then
+  usage
   exit 2
 fi
 
 agent_name=stirrup_agent
-scenario_ids=lite
+scenario_ids=tsfm_lite
 
 model_configs=(
   "litellm_proxy/gcp/gemini-3.6-flash high"
   "litellm_proxy/azure/gpt-5.6-sol max"
-  "tokenrouter/z-ai/glm-5.2 max"
-  "tokenrouter/MiniMax-M3 high"
   "litellm_proxy/aws/claude-opus-5 high"
-  "litellm_proxy/azure/DeepSeek-V4-Flash max"
-  "litellm_proxy/aws/gpt-oss-120b high"
   "litellm_proxy/aws/claude-sonnet-5 max"
+  "tokenrouter/MiniMax-M3 high"
+  "tokenrouter/moonshotai/kimi-k3 max"
+  "tokenrouter/z-ai/glm-5.3 max"
+  "tokenrouter/deepseek/deepseek-v4-flash max"
+  ""
+
 )
 
 for model_config in "${model_configs[@]}"; do
@@ -36,11 +73,11 @@ for model_config in "${model_configs[@]}"; do
     --agent_name "$agent_name" \
     --model-id "$model_id" \
     --reasoning-effort "$reasoning_effort" \
-    --trajectory-root "$leaderboard_dir/assetopsbench-trajectories" \
+    --trajectory-root "$trajectory_dir" \
     --reports-root "$leaderboard_dir/assetopsbench-reports" \
     --stirrup-workspace-root "$leaderboard_dir/assetopsbench-stirrup-workspaces" \
+    --skip-existing \
+    --continue-on-error \
     --preserve-workspaces
-    # --continue-on-error
-    # --skip-existing \
 
 done
diff --git a/benchmarks/scenario_suite/lite.yaml b/benchmarks/scenario_suite/lite.yaml
index 21e8f7b4..6278b127 100644
--- a/benchmarks/scenario_suite/lite.yaml
+++ b/benchmarks/scenario_suite/lite.yaml
@@ -2,35 +2,36 @@ car:
   - 151
   - 152
   - 153
-  - 156
-  - 167
+  - 157
+  - 159
+  - 160
   - 178
   - 180
   - 182
   - 183
+  - 189
   - 193
 fcc:
-  - 301
   - 303
+  - 304
   - 305
   - 308
-  - 314
-  - 316
+  - 312
   - 320
-  - 323
+  - 322
   - 325
-  - 327
 fmsr:
   - 902
   - 904
   - 905
   - 906
+  - 908
   - 915
   - 916
+  - 917
   - 920
   - 923
   - 928
-  - 932
 health:
   - 401
   - 402
@@ -42,11 +43,36 @@ health:
   - 408
   - 409
   - 410
-tsfm: []
+tsfm:
+  - 1001
+  - 1002
+  - 1003
+  - 1004
+  - 1005
+  - 1006
+  - 1007
+  - 1008
+  - 1009
+  - 1010
+  - 1011
+  - 1012
+  - 1013
+  - 1014
+  - 1015
+  - 1016
+  - 1017
+  - 1018
+  - 1019
+  - 1020
+  - 1021
+  - 1022
+  - 1023
+  - 1024
+  - 1025
 wosr:
-  - 5
-  - 9
+  - 11
   - 13
+  - 16
   - 20
   - 24
   - 31