From 1059cc9d664b4cd3c136ea5ef85977117966880f Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 5 Aug 2026 19:24:18 +0000 Subject: [PATCH 1/3] e2e: fail fast with a clear message when GH_TOKEN is dead The 2026-08-05 full-test-suite run failed before any LLM call because RDB_TESTER_PAT_TOKEN (set Feb 25) had expired: every gh call returned HTTP 401, the quota check spewed a bash syntax error trying to compare a JSON error body against an integer, and the failure surfaced as three confusing per-test errors instead of one diagnosis. Both e2e scripts now verify the token up front (gh api user) and exit with rotation instructions when it's invalid, and the shared check_graphql_quota copies guard against non-numeric parses. Co-Authored-By: Claude Fable 5 --- tests/e2e-security.sh | 15 +++++++++++++++ tests/e2e.sh | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/e2e-security.sh b/tests/e2e-security.sh index dba73e6..ab6c03d 100755 --- a/tests/e2e-security.sh +++ b/tests/e2e-security.sh @@ -58,6 +58,10 @@ check_graphql_quota() { fi remaining=$(echo "$result" | cut -d' ' -f1) reset_ts=$(echo "$result" | cut -d' ' -f2) + if ! [[ "$remaining" =~ ^[0-9]+$ ]]; then + echo "==> Warning: could not parse GraphQL rate limit response — proceeding anyway" + return + fi reset_time=$(python3 -c " import datetime ts = ${reset_ts} @@ -74,6 +78,17 @@ print(f'{local_t.strftime(\"%H:%M:%S\")} local / {utc_t.strftime(\"%H:%M:%S\")} echo "==> GraphQL quota: ${remaining} points remaining (resets ${reset_time})." } +# Fail fast with a clear message when GH_TOKEN is dead — an expired +# RDB_TESTER_PAT_TOKEN otherwise surfaces as confusing 401s on every +# issue-creation call after the suite has already started. +gh_login=$(gh api user --jq .login 2>/dev/null) || { + echo "ERROR: GH_TOKEN is invalid or expired (gh api user failed)." >&2 + echo "ERROR: In CI this token comes from secrets.RDB_TESTER_PAT_TOKEN —" >&2 + echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 + exit 1 +} +echo "==> Authenticated to GitHub as: ${gh_login}" + check_graphql_quota # --- Helpers --- diff --git a/tests/e2e.sh b/tests/e2e.sh index 4331471..250dabc 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -54,6 +54,10 @@ check_graphql_quota() { fi remaining=$(echo "$result" | cut -d' ' -f1) reset_ts=$(echo "$result" | cut -d' ' -f2) + if ! [[ "$remaining" =~ ^[0-9]+$ ]]; then + echo "==> Warning: could not parse GraphQL rate limit response — proceeding anyway" + return + fi reset_time=$(python3 -c " import datetime ts = ${reset_ts} @@ -70,6 +74,17 @@ print(f'{local_t.strftime(\"%H:%M:%S\")} local / {utc_t.strftime(\"%H:%M:%S\")} echo "==> GraphQL quota: ${remaining} points remaining (resets ${reset_time})." } +# Fail fast with a clear message when GH_TOKEN is dead — an expired +# RDB_TESTER_PAT_TOKEN otherwise surfaces as confusing 401s on every +# issue-creation call after the suite has already started. +gh_login=$(gh api user --jq .login 2>/dev/null) || { + echo "ERROR: GH_TOKEN is invalid or expired (gh api user failed)." >&2 + echo "ERROR: In CI this token comes from secrets.RDB_TESTER_PAT_TOKEN —" >&2 + echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 + exit 1 +} +echo "==> Authenticated to GitHub as: ${gh_login}" + check_graphql_quota # --- Test case definitions --- From f22ccdb417f057011846756f247d707ba6e642f0 Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Sat, 8 Aug 2026 04:49:00 +0000 Subject: [PATCH 2/3] CONTRIBUTING: document that RDB_TESTER_PAT_TOKEN is a gnovak PAT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The account table said the token belongs to the remote-dev-bot account, but that account was flagged by GitHub as a bot and authorized-trigger duty moved to gnovak — every e2e test issue on rdb-test is authored by gnovak. Documented the actual state, recommended a fine-grained PAT scoped to the test repo with the permissions the suite really needs, and noted the expiry failure mode (the 2026-08-05 suite failure). Co-Authored-By: Claude Fable 5 --- CONTRIBUTING.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4968bb..1df1aaa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,21 +24,25 @@ Three separate GitHub identities are used so each role is cleanly separated: | Account | Purpose | Credentials stored as | | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | `gnovak` | Repo owner. Used for normal development and testing. | `RDB_PAT_TOKEN` (on both repos), or GitHub App (`RDB_APP_ID` variable + `RDB_APP_PRIVATE_KEY` secret) | -| `remote-dev-bot` | Dedicated bot account. Collaborator on `remote-dev-bot-test`. Posts authorized test comments that trigger agent runs without attributing activity to `gnovak`. | `RDB_TESTER_PAT_TOKEN` (on remote-dev-bot) | +| `remote-dev-bot` | Dedicated bot account, collaborator on `remote-dev-bot-test`. **Historical:** it was flagged by GitHub as a bot account, so in practice `RDB_TESTER_PAT_TOKEN` is a `gnovak` PAT (verifiable: e2e test issues are authored by `gnovak`). | (none currently) | | `remote-dev-bot-tester` | Simulates an unauthorized external user. NOT a collaborator on any repo. | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` (on remote-dev-bot) | -### remote-dev-bot (bot account) details +### RDB_TESTER_PAT_TOKEN details -- Classic PAT with `public_repo` + `workflow` scopes, no expiration -- **`public_repo`** — create issues, post comments, open PRs in public repos - (standard e2e test flow) -- **`workflow`** — read/write `.github/workflows/` files; required when the - agent itself needs to modify workflow files in a PR (e.g., the dogfood use - case where rdb is editing its own workflow files). Without this scope, - writes to `.github/workflows/` fail with HTTP 404. -- Must be a collaborator on `remote-dev-bot-test` so the security gate allows - its trigger comments -- Keeps test activity out of `gnovak`'s GitHub contribution stats +In practice a PAT on the `gnovak` account (the dedicated `remote-dev-bot` +account was flagged by GitHub as a bot, so authorized-trigger duty moved to +the owner account; test activity therefore does show up in `gnovak`'s +contribution stats). + +- Recommended: fine-grained PAT scoped to `gnovak/remote-dev-bot-test` only, + with Contents read/write, Issues read/write, Pull requests read/write, and + Actions read/write (the suite monitors and cancels workflow runs). + A classic PAT with `repo` scope also works but grants far more than needed. +- Prefer a long or no expiration, and note the date somewhere visible — an + expired token surfaces as the e2e pre-flight failing with 401 (the + 2026-08-05 full-suite failure was exactly this, from a 90-day default). +- The token owner must pass the security gate on `remote-dev-bot-test` + (owner or collaborator) so its trigger comments start runs. ### remote-dev-bot-tester details @@ -81,7 +85,7 @@ Secrets stored on `gnovak/remote-dev-bot`: | `OPENAI_API_KEY` | OpenAI API key (for GPT models) | | `GEMINI_API_KEY` | Google AI API key (for Gemini models) | | `RDB_APP_PRIVATE_KEY` | GitHub App private key, for bot identity on comments/PRs. | -| `RDB_TESTER_PAT_TOKEN` | PAT for `remote-dev-bot` account (collaborator on rdb-test). Used by e2e tests to post authorized trigger comments. | +| `RDB_TESTER_PAT_TOKEN` | PAT for an account authorized on rdb-test (in practice `gnovak` — see account table). Used by e2e tests to create issues and post authorized trigger comments. | | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` | PAT for `remote-dev-bot-tester` (not a collaborator). Used by security e2e tests to verify unauthorized users are blocked. | Variables stored on `gnovak/remote-dev-bot`: From a311854718cf4b0fe2e3b42e0c6acdffe1c8e98a Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Sat, 8 Aug 2026 04:55:54 +0000 Subject: [PATCH 3/3] Rework token docs to intent-first; pre-flight prints expiry, checks both tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit documented the gnovak-PAT workaround as the standing setup. The bot-account arrangement is the intended design — the account is just flagged by GitHub right now (its profile 404s on the public API) pending a support conversation. Restore the intended setup as primary in CONTRIBUTING with a clearly-marked temporary-state note carrying the stopgap guidance and the restore path. Pre-flight hardening while here: print the token's expiration date (GitHub-Authentication-Token-Expiration header) alongside the login so future runs self-document how long the credential has left, and validate UNAUTHORIZED_PAT up front in e2e-security.sh — its failures otherwise surface only mid-gating-test. Co-Authored-By: Claude Fable 5 --- CONTRIBUTING.md | 45 ++++++++++++++++++++++++++----------------- tests/e2e-security.sh | 16 ++++++++++++++- tests/e2e.sh | 3 ++- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1df1aaa..969f1e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,25 +24,34 @@ Three separate GitHub identities are used so each role is cleanly separated: | Account | Purpose | Credentials stored as | | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | `gnovak` | Repo owner. Used for normal development and testing. | `RDB_PAT_TOKEN` (on both repos), or GitHub App (`RDB_APP_ID` variable + `RDB_APP_PRIVATE_KEY` secret) | -| `remote-dev-bot` | Dedicated bot account, collaborator on `remote-dev-bot-test`. **Historical:** it was flagged by GitHub as a bot account, so in practice `RDB_TESTER_PAT_TOKEN` is a `gnovak` PAT (verifiable: e2e test issues are authored by `gnovak`). | (none currently) | +| `remote-dev-bot` | Dedicated bot account. Collaborator on `remote-dev-bot-test`. Posts authorized test comments that trigger agent runs without attributing activity to `gnovak`. | `RDB_TESTER_PAT_TOKEN` (on remote-dev-bot) — but see the temporary-state note below | | `remote-dev-bot-tester` | Simulates an unauthorized external user. NOT a collaborator on any repo. | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` (on remote-dev-bot) | -### RDB_TESTER_PAT_TOKEN details - -In practice a PAT on the `gnovak` account (the dedicated `remote-dev-bot` -account was flagged by GitHub as a bot, so authorized-trigger duty moved to -the owner account; test activity therefore does show up in `gnovak`'s -contribution stats). - -- Recommended: fine-grained PAT scoped to `gnovak/remote-dev-bot-test` only, - with Contents read/write, Issues read/write, Pull requests read/write, and - Actions read/write (the suite monitors and cancels workflow runs). - A classic PAT with `repo` scope also works but grants far more than needed. -- Prefer a long or no expiration, and note the date somewhere visible — an - expired token surfaces as the e2e pre-flight failing with 401 (the - 2026-08-05 full-suite failure was exactly this, from a 90-day default). -- The token owner must pass the security gate on `remote-dev-bot-test` - (owner or collaborator) so its trigger comments start runs. +### remote-dev-bot (bot account) details + +The intended setup: + +- Classic PAT with `public_repo` + `workflow` scopes, no expiration +- **`public_repo`** — create issues, post comments, open PRs in public repos + (standard e2e test flow) +- **`workflow`** — read/write `.github/workflows/` files; required when the + agent itself needs to modify workflow files in a PR (e.g., the dogfood use + case where rdb is editing its own workflow files). Without this scope, + writes to `.github/workflows/` fail with HTTP 404. +- Must be a collaborator on `remote-dev-bot-test` so the security gate allows + its trigger comments +- Keeps test activity out of `gnovak`'s GitHub contribution stats + +> **Temporary state (as of 2026-08):** the `remote-dev-bot` account is +> flagged by GitHub (its profile 404s on the public API), so tokens cannot +> be minted on it until that's resolved with GitHub support. Until then, +> `RDB_TESTER_PAT_TOKEN` holds a PAT minted on `gnovak` instead (verifiable: +> e2e test issues on rdb-test are authored by `gnovak`). If minting the +> stopgap on `gnovak`, prefer a fine-grained PAT scoped to +> `gnovak/remote-dev-bot-test` only (Contents/Issues/Pull requests +> read-write + Actions read-write) with a long expiry — the 2026-08-05 +> full-suite failure was a 90-day stopgap token quietly expiring. Once the +> bot account is restored, move the token back per the intended setup above. ### remote-dev-bot-tester details @@ -85,7 +94,7 @@ Secrets stored on `gnovak/remote-dev-bot`: | `OPENAI_API_KEY` | OpenAI API key (for GPT models) | | `GEMINI_API_KEY` | Google AI API key (for Gemini models) | | `RDB_APP_PRIVATE_KEY` | GitHub App private key, for bot identity on comments/PRs. | -| `RDB_TESTER_PAT_TOKEN` | PAT for an account authorized on rdb-test (in practice `gnovak` — see account table). Used by e2e tests to create issues and post authorized trigger comments. | +| `RDB_TESTER_PAT_TOKEN` | PAT for `remote-dev-bot` account (collaborator on rdb-test; temporarily a `gnovak` PAT — see account details). Used by e2e tests to post authorized trigger comments. | | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` | PAT for `remote-dev-bot-tester` (not a collaborator). Used by security e2e tests to verify unauthorized users are blocked. | Variables stored on `gnovak/remote-dev-bot`: diff --git a/tests/e2e-security.sh b/tests/e2e-security.sh index ab6c03d..f7e3ca5 100755 --- a/tests/e2e-security.sh +++ b/tests/e2e-security.sh @@ -87,7 +87,21 @@ gh_login=$(gh api user --jq .login 2>/dev/null) || { echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 exit 1 } -echo "==> Authenticated to GitHub as: ${gh_login}" +gh_exp=$(gh api user -i 2>/dev/null | grep -i "^github-authentication-token-expiration:" | cut -d" " -f2- | tr -d "\r") +echo "==> Authenticated to GitHub as: ${gh_login}${gh_exp:+ (token expires: ${gh_exp})}" + +# Validate the unauthorized-user token too, when present — its failures +# otherwise surface only mid-gating-test. +if [[ -n "${UNAUTHORIZED_PAT:-}" ]]; then + ua_login=$(curl -fs -H "Authorization: token ${UNAUTHORIZED_PAT}" \ + https://api.github.com/user 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['login'])" 2>/dev/null) || true + if [[ -z "${ua_login:-}" ]]; then + echo "ERROR: UNAUTHORIZED_PAT is invalid or expired (api.github.com/user failed)." >&2 + echo "ERROR: In CI this is secrets.RDB_TESTER_UNAUTHORIZED_PAT_TOKEN (remote-dev-bot-tester account)." >&2 + exit 1 + fi + echo "==> Unauthorized-user token authenticates as: ${ua_login}" +fi check_graphql_quota diff --git a/tests/e2e.sh b/tests/e2e.sh index 250dabc..2c2b255 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -83,7 +83,8 @@ gh_login=$(gh api user --jq .login 2>/dev/null) || { echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 exit 1 } -echo "==> Authenticated to GitHub as: ${gh_login}" +gh_exp=$(gh api user -i 2>/dev/null | grep -i "^github-authentication-token-expiration:" | cut -d" " -f2- | tr -d "\r") +echo "==> Authenticated to GitHub as: ${gh_login}${gh_exp:+ (token expires: ${gh_exp})}" check_graphql_quota