From 18c6b3c6f46acd25368bbb3b7bfbed64a964a31c Mon Sep 17 00:00:00 2001 From: thisisjun786 <259586770+thisisjun786@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:41:12 +0900 Subject: [PATCH 1/3] feat(relay/delivery): delivery, intent, admission, reconcile and recover Port todo 21 of crw-go-port: delivery emit/deliver/claim/ack/verdict, intents and markers, admission, reconcile and recover, host-loss and unknown-send handling, ACK/re-review/recovery paths, with one Go test per contract/behaviour property (216 ids, 213 ported, 3 python-internal) plus the six attempt-message cases handed over from todo 18, each compared with real Python output. Subsets of faults (todo 22) and mergeturn (todo 26) are marked with their owners. --- docs/port/test-map.md | 90 +- internal/contracttest/cli.go | 2 + .../cli/attempt_message_atomicity_test.go | 157 +++ internal/relay/cli/registry.go | 47 +- internal/relay/delivery/ack.go | 848 +++++++++++++ .../relay/delivery/ackreconcile_a_test.go | 198 +++ .../relay/delivery/ackreconcile_b_test.go | 216 ++++ .../relay/delivery/ackreconcile_c_test.go | 228 ++++ internal/relay/delivery/adapter.go | 57 + internal/relay/delivery/anchor_test.go | 268 ++++ internal/relay/delivery/anchors.go | 41 + .../attempt_message_atomicity_test.go | 164 +++ internal/relay/delivery/bridge_reads.go | 206 ++++ internal/relay/delivery/carried_test.go | 275 +++++ internal/relay/delivery/cli.go | 567 +++++++++ internal/relay/delivery/cli_emit.go | 122 ++ internal/relay/delivery/cli_test.go | 197 +++ internal/relay/delivery/clock.go | 50 + internal/relay/delivery/criteria.go | 487 ++++++++ internal/relay/delivery/criteria_test.go | 186 +++ internal/relay/delivery/currency.go | 195 +++ internal/relay/delivery/currency_ack_test.go | 149 +++ internal/relay/delivery/currency_test.go | 333 +++++ internal/relay/delivery/delivery_a_test.go | 266 ++++ internal/relay/delivery/delivery_b_test.go | 332 +++++ internal/relay/delivery/delivery_c_test.go | 307 +++++ internal/relay/delivery/delivery_d_test.go | 181 +++ .../relay/delivery/delivery_relation_test.go | 104 ++ internal/relay/delivery/errors.go | 50 + internal/relay/delivery/fakehost_test.go | 299 +++++ internal/relay/delivery/harness_test.go | 530 ++++++++ internal/relay/delivery/hostadapter.go | 199 +++ internal/relay/delivery/hostcheck.go | 422 +++++++ internal/relay/delivery/hostloss.go | 336 +++++ internal/relay/delivery/hostloss_a_test.go | 364 ++++++ internal/relay/delivery/hostloss_b_test.go | 590 +++++++++ internal/relay/delivery/hostloss_c_test.go | 606 +++++++++ internal/relay/delivery/hostloss_d_test.go | 250 ++++ .../relay/delivery/hostloss_faults_test.go | 54 + .../relay/delivery/hostloss_harness_test.go | 805 ++++++++++++ internal/relay/delivery/intent.go | 945 ++++++++++++++ internal/relay/delivery/intent_cli.go | 541 ++++++++ internal/relay/delivery/intent_cli_test.go | 130 ++ internal/relay/delivery/intent_test.go | 513 ++++++++ internal/relay/delivery/lifecycle.go | 104 ++ internal/relay/delivery/literals_test.go | 121 ++ internal/relay/delivery/main_test.go | 24 + internal/relay/delivery/marker.go | 503 ++++++++ internal/relay/delivery/marker_ops_test.go | 442 +++++++ internal/relay/delivery/marker_test.go | 188 +++ internal/relay/delivery/multiparent_test.go | 290 +++++ .../relay/delivery/onrequest_adapter_test.go | 147 +++ internal/relay/delivery/onrequest_test.go | 133 ++ .../relay/delivery/opengeneration_test.go | 30 + internal/relay/delivery/policy.go | 94 ++ internal/relay/delivery/preview.go | 443 ------- internal/relay/delivery/preview_test.go | 3 +- internal/relay/delivery/projection.go | 123 ++ internal/relay/delivery/pyjson.go | 243 ++++ internal/relay/delivery/qa_test.go | 38 + internal/relay/delivery/race_recovery_test.go | 249 ++++ internal/relay/delivery/reconcile.go | 649 ++++++++++ internal/relay/delivery/reconcile_pass.go | 244 ++++ .../delivery/registration_hold_live_test.go | 50 + internal/relay/delivery/relationship.go | 92 ++ internal/relay/delivery/render.go | 367 ++++++ internal/relay/delivery/rereview_test.go | 487 ++++++++ internal/relay/delivery/revision_test.go | 211 ++++ internal/relay/delivery/rows.go | 122 ++ internal/relay/delivery/scheduler.go | 145 +++ internal/relay/delivery/scope.go | 77 ++ internal/relay/delivery/selection.go | 117 ++ internal/relay/delivery/selection_test.go | 75 ++ internal/relay/delivery/service.go | 1095 +++++++++++++++++ internal/relay/delivery/services_test.go | 47 + internal/relay/delivery/settings.go | 345 ++++++ internal/relay/delivery/snapshot.go | 219 ++++ internal/relay/delivery/supersession_test.go | 292 +++++ internal/relay/delivery/sync.go | 148 +++ internal/relay/delivery/testdata/capture.py | 100 ++ internal/relay/delivery/testdata/cliseed.py | 12 + .../relay/delivery/testdata/drl_answers.json | 14 + .../relay/delivery/testdata/grantstage.py | 58 + internal/relay/delivery/testdata/markerops.py | 215 ++++ .../relay/delivery/testdata/ordadapter.py | 54 + .../relay/delivery/testdata/pyscenario.py | 56 + .../relay/delivery/testdata/scenarios/_vcu.py | 25 + .../relay/delivery/testdata/scenarios/anb.py | 78 ++ .../relay/delivery/testdata/scenarios/crr.py | 40 + .../delivery/testdata/scenarios/del01.py | 4 + .../delivery/testdata/scenarios/del02.py | 4 + .../delivery/testdata/scenarios/del03.py | 28 + .../delivery/testdata/scenarios/del04.py | 3 + .../delivery/testdata/scenarios/del05.py | 5 + .../delivery/testdata/scenarios/del06.py | 3 + .../delivery/testdata/scenarios/del07.py | 5 + .../delivery/testdata/scenarios/del08.py | 4 + .../delivery/testdata/scenarios/del10.py | 8 + .../delivery/testdata/scenarios/del11.py | 6 + .../delivery/testdata/scenarios/del12.py | 12 + .../delivery/testdata/scenarios/del13.py | 28 + .../delivery/testdata/scenarios/del14.py | 11 + .../delivery/testdata/scenarios/del16.py | 9 + .../delivery/testdata/scenarios/del17.py | 15 + .../delivery/testdata/scenarios/del18.py | 3 + .../delivery/testdata/scenarios/del20.py | 10 + .../delivery/testdata/scenarios/del21.py | 17 + .../delivery/testdata/scenarios/del22.py | 7 + .../delivery/testdata/scenarios/del24.py | 15 + .../delivery/testdata/scenarios/del25.py | 7 + .../delivery/testdata/scenarios/del26.py | 18 + .../delivery/testdata/scenarios/del27.py | 17 + .../delivery/testdata/scenarios/del30.py | 8 + .../delivery/testdata/scenarios/del31.py | 25 + .../delivery/testdata/scenarios/del32.py | 7 + .../delivery/testdata/scenarios/del33.py | 9 + .../delivery/testdata/scenarios/del34.py | 17 + .../delivery/testdata/scenarios/del35.py | 21 + .../relay/delivery/testdata/scenarios/drl.py | 14 + .../relay/delivery/testdata/scenarios/mpi.py | 84 ++ .../relay/delivery/testdata/scenarios/ogi.py | 8 + .../relay/delivery/testdata/scenarios/ord.py | 41 + .../relay/delivery/testdata/scenarios/qa.py | 9 + .../relay/delivery/testdata/scenarios/rvr.py | 75 ++ .../relay/delivery/testdata/scenarios/sup.py | 95 ++ .../relay/delivery/testdata/scenarios/vcu.py | 63 + .../delivery/testdata/scenarios/vcu_ack.py | 37 + internal/relay/delivery/transport.go | 193 +++ internal/relay/delivery/unknownsend_a_test.go | 637 ++++++++++ internal/relay/delivery/unknownsend_b_test.go | 598 +++++++++ .../relay/delivery/unknownsend_grant_test.go | 145 +++ internal/relay/delivery/value.go | 218 ---- internal/relay/delivery/verify.go | 228 ++++ internal/relay/faults/json.go | 176 +++ internal/relay/faults/ledger.go | 585 +++++++++ internal/relay/faults/sweep.go | 580 +++++++++ internal/relay/mergeturn/grant.go | 163 +++ internal/relay/store/receipt_admission.go | 25 +- internal/relay/store/receipt_continuation.go | 168 +++ internal/relay/store/receipt_intake.go | 33 +- internal/relay/store/records.go | 8 + internal/relay/store/registration_hold.go | 49 + 142 files changed, 25364 insertions(+), 710 deletions(-) create mode 100644 internal/relay/cli/attempt_message_atomicity_test.go create mode 100644 internal/relay/delivery/ack.go create mode 100644 internal/relay/delivery/ackreconcile_a_test.go create mode 100644 internal/relay/delivery/ackreconcile_b_test.go create mode 100644 internal/relay/delivery/ackreconcile_c_test.go create mode 100644 internal/relay/delivery/adapter.go create mode 100644 internal/relay/delivery/anchor_test.go create mode 100644 internal/relay/delivery/anchors.go create mode 100644 internal/relay/delivery/attempt_message_atomicity_test.go create mode 100644 internal/relay/delivery/bridge_reads.go create mode 100644 internal/relay/delivery/carried_test.go create mode 100644 internal/relay/delivery/cli.go create mode 100644 internal/relay/delivery/cli_emit.go create mode 100644 internal/relay/delivery/cli_test.go create mode 100644 internal/relay/delivery/clock.go create mode 100644 internal/relay/delivery/criteria.go create mode 100644 internal/relay/delivery/criteria_test.go create mode 100644 internal/relay/delivery/currency.go create mode 100644 internal/relay/delivery/currency_ack_test.go create mode 100644 internal/relay/delivery/currency_test.go create mode 100644 internal/relay/delivery/delivery_a_test.go create mode 100644 internal/relay/delivery/delivery_b_test.go create mode 100644 internal/relay/delivery/delivery_c_test.go create mode 100644 internal/relay/delivery/delivery_d_test.go create mode 100644 internal/relay/delivery/delivery_relation_test.go create mode 100644 internal/relay/delivery/errors.go create mode 100644 internal/relay/delivery/fakehost_test.go create mode 100644 internal/relay/delivery/harness_test.go create mode 100644 internal/relay/delivery/hostadapter.go create mode 100644 internal/relay/delivery/hostcheck.go create mode 100644 internal/relay/delivery/hostloss.go create mode 100644 internal/relay/delivery/hostloss_a_test.go create mode 100644 internal/relay/delivery/hostloss_b_test.go create mode 100644 internal/relay/delivery/hostloss_c_test.go create mode 100644 internal/relay/delivery/hostloss_d_test.go create mode 100644 internal/relay/delivery/hostloss_faults_test.go create mode 100644 internal/relay/delivery/hostloss_harness_test.go create mode 100644 internal/relay/delivery/intent.go create mode 100644 internal/relay/delivery/intent_cli.go create mode 100644 internal/relay/delivery/intent_cli_test.go create mode 100644 internal/relay/delivery/intent_test.go create mode 100644 internal/relay/delivery/lifecycle.go create mode 100644 internal/relay/delivery/literals_test.go create mode 100644 internal/relay/delivery/main_test.go create mode 100644 internal/relay/delivery/marker.go create mode 100644 internal/relay/delivery/marker_ops_test.go create mode 100644 internal/relay/delivery/marker_test.go create mode 100644 internal/relay/delivery/multiparent_test.go create mode 100644 internal/relay/delivery/onrequest_adapter_test.go create mode 100644 internal/relay/delivery/onrequest_test.go create mode 100644 internal/relay/delivery/opengeneration_test.go create mode 100644 internal/relay/delivery/policy.go delete mode 100644 internal/relay/delivery/preview.go create mode 100644 internal/relay/delivery/projection.go create mode 100644 internal/relay/delivery/pyjson.go create mode 100644 internal/relay/delivery/qa_test.go create mode 100644 internal/relay/delivery/race_recovery_test.go create mode 100644 internal/relay/delivery/reconcile.go create mode 100644 internal/relay/delivery/reconcile_pass.go create mode 100644 internal/relay/delivery/registration_hold_live_test.go create mode 100644 internal/relay/delivery/relationship.go create mode 100644 internal/relay/delivery/render.go create mode 100644 internal/relay/delivery/rereview_test.go create mode 100644 internal/relay/delivery/revision_test.go create mode 100644 internal/relay/delivery/rows.go create mode 100644 internal/relay/delivery/scheduler.go create mode 100644 internal/relay/delivery/scope.go create mode 100644 internal/relay/delivery/selection.go create mode 100644 internal/relay/delivery/selection_test.go create mode 100644 internal/relay/delivery/service.go create mode 100644 internal/relay/delivery/services_test.go create mode 100644 internal/relay/delivery/settings.go create mode 100644 internal/relay/delivery/snapshot.go create mode 100644 internal/relay/delivery/supersession_test.go create mode 100644 internal/relay/delivery/sync.go create mode 100644 internal/relay/delivery/testdata/capture.py create mode 100644 internal/relay/delivery/testdata/cliseed.py create mode 100644 internal/relay/delivery/testdata/drl_answers.json create mode 100644 internal/relay/delivery/testdata/grantstage.py create mode 100644 internal/relay/delivery/testdata/markerops.py create mode 100644 internal/relay/delivery/testdata/ordadapter.py create mode 100644 internal/relay/delivery/testdata/pyscenario.py create mode 100644 internal/relay/delivery/testdata/scenarios/_vcu.py create mode 100644 internal/relay/delivery/testdata/scenarios/anb.py create mode 100644 internal/relay/delivery/testdata/scenarios/crr.py create mode 100644 internal/relay/delivery/testdata/scenarios/del01.py create mode 100644 internal/relay/delivery/testdata/scenarios/del02.py create mode 100644 internal/relay/delivery/testdata/scenarios/del03.py create mode 100644 internal/relay/delivery/testdata/scenarios/del04.py create mode 100644 internal/relay/delivery/testdata/scenarios/del05.py create mode 100644 internal/relay/delivery/testdata/scenarios/del06.py create mode 100644 internal/relay/delivery/testdata/scenarios/del07.py create mode 100644 internal/relay/delivery/testdata/scenarios/del08.py create mode 100644 internal/relay/delivery/testdata/scenarios/del10.py create mode 100644 internal/relay/delivery/testdata/scenarios/del11.py create mode 100644 internal/relay/delivery/testdata/scenarios/del12.py create mode 100644 internal/relay/delivery/testdata/scenarios/del13.py create mode 100644 internal/relay/delivery/testdata/scenarios/del14.py create mode 100644 internal/relay/delivery/testdata/scenarios/del16.py create mode 100644 internal/relay/delivery/testdata/scenarios/del17.py create mode 100644 internal/relay/delivery/testdata/scenarios/del18.py create mode 100644 internal/relay/delivery/testdata/scenarios/del20.py create mode 100644 internal/relay/delivery/testdata/scenarios/del21.py create mode 100644 internal/relay/delivery/testdata/scenarios/del22.py create mode 100644 internal/relay/delivery/testdata/scenarios/del24.py create mode 100644 internal/relay/delivery/testdata/scenarios/del25.py create mode 100644 internal/relay/delivery/testdata/scenarios/del26.py create mode 100644 internal/relay/delivery/testdata/scenarios/del27.py create mode 100644 internal/relay/delivery/testdata/scenarios/del30.py create mode 100644 internal/relay/delivery/testdata/scenarios/del31.py create mode 100644 internal/relay/delivery/testdata/scenarios/del32.py create mode 100644 internal/relay/delivery/testdata/scenarios/del33.py create mode 100644 internal/relay/delivery/testdata/scenarios/del34.py create mode 100644 internal/relay/delivery/testdata/scenarios/del35.py create mode 100644 internal/relay/delivery/testdata/scenarios/drl.py create mode 100644 internal/relay/delivery/testdata/scenarios/mpi.py create mode 100644 internal/relay/delivery/testdata/scenarios/ogi.py create mode 100644 internal/relay/delivery/testdata/scenarios/ord.py create mode 100644 internal/relay/delivery/testdata/scenarios/qa.py create mode 100644 internal/relay/delivery/testdata/scenarios/rvr.py create mode 100644 internal/relay/delivery/testdata/scenarios/sup.py create mode 100644 internal/relay/delivery/testdata/scenarios/vcu.py create mode 100644 internal/relay/delivery/testdata/scenarios/vcu_ack.py create mode 100644 internal/relay/delivery/transport.go create mode 100644 internal/relay/delivery/unknownsend_a_test.go create mode 100644 internal/relay/delivery/unknownsend_b_test.go create mode 100644 internal/relay/delivery/unknownsend_grant_test.go delete mode 100644 internal/relay/delivery/value.go create mode 100644 internal/relay/delivery/verify.go create mode 100644 internal/relay/faults/json.go create mode 100644 internal/relay/faults/ledger.go create mode 100644 internal/relay/faults/sweep.go create mode 100644 internal/relay/mergeturn/grant.go create mode 100644 internal/relay/store/receipt_continuation.go create mode 100644 internal/relay/store/registration_hold.go diff --git a/docs/port/test-map.md b/docs/port/test-map.md index 147910ba..c077ea04 100644 --- a/docs/port/test-map.md +++ b/docs/port/test-map.md @@ -65,6 +65,94 @@ Go tests in `internal/bridge`, each reviewed against its Python property: The remaining MCP case is deferred to todo 16's "Expose the bridge as an MCP stdio server and wire `crw bridge`" and "QA scenarios: happy = stdio round trip create_thread against fakehost via an MCP client in Go test". Property parity is recorded row by row in the todo-15 ledger below. The todo-15 bridge fakehost work is separate from the 36 execution and 23 settings package properties counted under todo 14 in the file rows below. +## Todo 21 part A delivery property progress (2026-09-26) + +Ported by property (Jun, 2026-09-26). Go tests in `internal/relay/delivery`, named +`Test_...` after `.omo/ulw-execute/todo21-properties.md`; each runs the same scenario through +the real Python package over the same fixture tree and compares records, refusals and every table +row with Python's. + +| Python file | properties | Go tests | python-internal (not ported) | +| --- | ---: | ---: | --- | +| `test_delivery.py` | 35 | 35 | - | +| `test_delivery_relation.py` | 5 | 4 | DRL-0 (drift guard between two Python spellings; Go has one constant) | +| `test_supersession.py` | 8 | 8 | - | +| `test_multi_parent_isolation.py` | 5 | 5 | - | +| `test_on_request_delivery.py` | 9 | 9 | - | +| `test_anchor_binding.py` | 8 | 8 | - | +| `test_criteria_registration.py` | 7 | 7 | - | +| `test_revision_roundtrip.py` | 5 | 5 | - | +| `test_verification_currency.py` | 13 | 12 | VCU-5 (`inspect.signature`; Go `RecordVerdict` has no bypass parameter) | +| **Total** | **95** | **93** | **2** | + +The twelve delivery commands (`emit`, `deliver`, `reconcile`, `recover`, `claim`, `ack-proof`, +`ack`, `verdict`, `criteria-register`, `criteria-show`, `revision-head`, `verify-acks`) answer +byte for byte like Python (`TestCLI_every_delivery_command_answers_byte_for_byte_like_python`); +`deliver`, `reconcile`, `recover` and `verify-acks` with `--socket` need the host adapter +(todo 28). + +## Todo 21 part B1 intent and marker property progress (2026-09-26) + +Go tests in `internal/relay/delivery`, named `Test_...` after +`.omo/ulw-execute/todo21-properties.md`; each runs one JSON op list through the real `marker.py` / +`intent.py` (`testdata/markerops.py`) and the Go port over the same tree and compares the whole +answers (records, refusal reason and detail, derived states, fact reads). + +| Python file | properties | Go tests | python-internal (not ported) | +| --- | ---: | ---: | --- | +| `test_intent.py` | 23 | 23 | - | +| `test_marker.py` | 8 | 8 | - | +| **Total** | **31** | **31** | **0** | + +The eight marker commands (`intent-declare`, `intent-attempt`, `intent-bind`, `intent-register`, +`intent-claim`, `intent-disposition`, `intent-resolve`, `intent-show`), with the store records +`intent-claim` and `intent-disposition` mirror (declarations.py), answer byte for byte like Python +(`TestCLI_every_intent_command_answers_byte_for_byte_like_python`); `guard-evaluate` is todo 33. + +## Todo 21 part B2 host-loss and unknown-send property progress (2026-09-26) + +Go tests in `internal/relay/delivery`, named `Test21__...` after +`.omo/ulw-execute/todo21-properties.md`. `testdata/capture.py` runs every Python test of the file, +each in its own tree, and records every value it asserts; the Go twin of each test runs the same +steps in the same tree and must produce the same values, delivery tables, `delivery_stalled` +fault rows and host sends. + +| Python file | properties | Go tests | python-internal (not ported) | +| --- | ---: | ---: | --- | +| `test_host_lost_turn.py` | 29 | 29 | - | +| `test_unknown_send_lost.py` | 23 | 22 | USL-18 (`assertIs` on `cli.Services` wiring; covered by USL-15/17) | +| **Total** | **52** | **51** | **1** | + +The fault sweep/ledger (todo 22) and the merge-turn grant rule (todo 26) these properties read are +ported as marked subsets in `internal/relay/faults` and `internal/relay/mergeturn`; the recipient +reads of the bridge adapter (todo 28) as `delivery.BridgeReads`. + +## Todo 21 part C acknowledgement, re-review and recovery property progress (2026-09-26) + +Same method as part B2 (`testdata/capture.py` + the Go twin of each Python test, every asserted +value and the delivery tables compared; the re-review tests also compare the review tables). + +| Python file | properties | Go tests | python-internal (not ported) | +| --- | ---: | ---: | --- | +| `test_ack_reconcile.py` | 23 | 23 | - | +| `test_ack_disposition_race.py` | 3 | 3 | - | +| `test_rereview_deadlock.py` | 9 | 9 | - | +| `test_recovery_negatives.py` | 3 | 3 | - | +| **Total** | **38** | **38** | **0** | + +Carried from todo 25A into `internal/relay/delivery`: `test_cli.py` CLI-5, CLI-7, CLI-9, CLI-21, +CLI-38 (`Test25_CLI*`, whole stdout against the Python command) and +`test_registration_contention.py` RCT-1, RCT-4 for their `intent.bind` marker half (`Test25_RCT*`; +the guard-evaluate half is todo 33). + +Carried from todo 18 into todo 21: `test_attempt_message_atomicity.py` AMA-1..AMA-6 +(`Test21_AMA1..6` in `internal/relay/delivery/attempt_message_atomicity_test.go`, +with `cmd_show` AMA-4..AMA-6 in `internal/relay/cli/attempt_message_atomicity_test.go`). +The Go mirrors compare the Python tests' captured assertions via `testdata/capture.py`; +AMA-6 pins oldest-first order independently in both Service.AttemptMessages and CLI show. +`Test21_RegistrationHold_refuses_resolved_live_state` pins the registration hold's +write admission against a symlink into a temporary live-state-shaped directory. + ## Files | path | tests | class | family | fixtures | owner | destination | coupling | @@ -73,7 +161,7 @@ The remaining MCP case is deferred to todo 16's "Expose the bridge as an MCP std | `packages/codex-session-relay/tests/test_ack_reconcile.py` | 40 | B | ACK, verdicts, reconciliation and restart recovery | - | todo 21 / CRW-153 | go-test: `internal/relay/delivery` (todo 21) | - | | `packages/codex-session-relay/tests/test_anchor_binding.py` | 14 | B | an anchor binds on every route to dispatched | - | todo 21 / CRW-153 | go-test: `internal/relay/delivery` (todo 21) | - | | `packages/codex-session-relay/tests/test_assignment.py` | 58 | B | assignment ledger: one issue, one responsible child | - | todo 25 / CRW-154 | go-test: `internal/relay/registry` (todo 25) + go-test: `internal/relay/store` (todo 19: typed queries, guard-index refusals and Python-store parity for the tables this file writes) | - | -| `packages/codex-session-relay/tests/test_attempt_message_atomicity.py` | 13 | B | a send carries the message its attempt froze | - | todo 18 / CRW-152 | go-test: `internal/relay/store` (todo 18); DeliveryService pre-claim seam, reconciliation and cmd_show preview halves of 6 cases -> todo 21 `internal/relay/delivery` | - | +| `packages/codex-session-relay/tests/test_attempt_message_atomicity.py` | 13 | B | a send carries the message its attempt froze | - | todo 18 / CRW-152 | go-test: `internal/relay/store` (todo 18); DeliveryService pre-claim seam, reconciliation and cmd_show preview halves of 6 cases -> todo 21 `internal/relay/delivery` and `internal/relay/cli` (`Test21_AMA1..6`, plus CLI AMA-4..6) | - | | `packages/codex-session-relay/tests/test_bridge_adapter.py` | 75 | B | bridge adapter logic over an injected RPC surface | `codex_session_relay.fakehost` | todo 28 / CRW-154 | go-test: bridge adapter package (todo 28) | - | | `packages/codex-session-relay/tests/test_bridge_load_roots.py` | 12 | B | a parent loaded by another task's bridge is still reached (CRW-235) | - | todo 28 / CRW-154 | go-test: bridge adapter package (todo 28) | - | | `packages/codex-session-relay/tests/test_capacity.py` | 30 | B | capacity counting and refusal to infer | - | todo 27 / CRW-154 | go-test: capacity package (todo 27) + go-test: `internal/relay/store` (todo 19: typed queries, guard-index refusals and Python-store parity for the tables this file writes) | - | diff --git a/internal/contracttest/cli.go b/internal/contracttest/cli.go index a23cf0f1..487e480e 100644 --- a/internal/contracttest/cli.go +++ b/internal/contracttest/cli.go @@ -50,6 +50,8 @@ func runCLI(t *testing.T, scenario Scenario) (map[string]any, error) { } // A scenario that drives a relay command the Go build has not registered yet is not ported: // it is counted as a skip (a failure under CRW_CONTRACT_STRICT=1), never run against usage. + // relaycli.Registered is the union of both gates: the delivery commands (todo 21) and the + // diagnostics commands of internal/relay/cli (todo 20: doctor, status, show, store-*). for _, step := range steps { if argv, _ := step["argv"].([]any); len(argv) > 0 { if name, ok := argv[0].(string); ok && !relaycli.Registered(name) { diff --git a/internal/relay/cli/attempt_message_atomicity_test.go b/internal/relay/cli/attempt_message_atomicity_test.go new file mode 100644 index 00000000..7eca6af4 --- /dev/null +++ b/internal/relay/cli/attempt_message_atomicity_test.go @@ -0,0 +1,157 @@ +package cli_test + +import ( + "database/sql" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + _ "modernc.org/sqlite" +) + +type amaCapture struct { + Captures []any `json:"captures"` + Problems []string `json:"problems"` +} + +func amaTree(t *testing.T, method string) (string, amaCapture) { + t.Helper() + amaRoot := t.TempDir() + home := t.TempDir() + cmd := exec.Command("uv", "run", "--no-sync", "python", filepath.Join(repositoryRoot(t), "internal/relay/delivery/testdata/capture.py"), amaRoot, "test_attempt_message_atomicity") + cmd.Dir = filepath.Join(repositoryRoot(t), "packages/codex-session-relay") + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "state"), "XDG_DATA_HOME="+filepath.Join(home, "data"), "XDG_CONFIG_HOME="+filepath.Join(home, "config"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, "PYTHONPATH="+filepath.Join(repositoryRoot(t), "packages/codex-session-relay/src")+":"+filepath.Join(repositoryRoot(t), "packages/codex-session-relay")) + if output, err := cmd.CombinedOutput(); err != nil { + t.Fatal(&captureFailure{err, string(output)}) + } + tree := filepath.Join(amaRoot, "AttemptMessageAtomicity."+method) + raw, err := os.ReadFile(filepath.Join(tree, "capture.json")) + if err != nil { + t.Fatal(err) + } + var captured amaCapture + if err := json.Unmarshal(raw, &captured); err != nil { + t.Fatal(err) + } + if len(captured.Problems) > 0 { + t.Fatal(captured.Problems) + } + return tree, captured +} + +type captureFailure struct { + err error + output string +} + +func (e *captureFailure) Error() string { return e.err.Error() + "\n" + e.output } +func amaShow(t *testing.T, tree string, event string) map[string]any { + t.Helper() + home := pythonHome(t) + state := filepath.Join(tree, "state") + result := golang(t, home, "--state", state, "show", "--event", event, "--message") + if result.code != 0 { + t.Fatalf("show: %+v", result) + } + return decode(t, result.stdout) +} +func amaEvent(t *testing.T, tree string) string { + t.Helper() + // The Python case has exactly one event. Read its ID from its captured store. + state := filepath.Join(tree, "state", "relay.sqlite3") + db, err := sql.Open("sqlite", state) + if err != nil { + t.Fatal(err) + } + defer db.Close() + var event string + if err := db.QueryRow("SELECT event_id FROM events").Scan(&event); err != nil { + t.Fatal(err) + } + return event +} +func amaTokenCLI(t *testing.T, value any) string { + t.Helper() + for _, line := range strings.Split(value.(string), "\n") { + if strings.HasPrefix(line, "requestId: ") { + return strings.TrimPrefix(line, "requestId: ") + } + } + t.Fatal("message carries no requestId") + return "" +} +func amaCompare(t *testing.T, got, want []any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("assertions Go=%v Python=%v", got, want) + } +} +func Test21_AMA4_show_message_returns_the_sent_attempt_after_a_send(t *testing.T) { + tree, py := amaTree(t, "test_show_message_returns_the_sent_attempt_after_a_send") + payload := amaShow(t, tree, amaEvent(t, tree)) + entries := payload["attemptMessages"].([]any) + e := entries[0].(map[string]any) + _, preview := payload["previewMessage"] + amaCompare(t, []any{float64(len(entries)), e["requestId"], e["status"], amaTokenCLI(t, e["message"]), preview}, py.Captures) +} +func Test21_AMA5_show_message_offers_a_preview_only_before_anything_is_prepared(t *testing.T) { + tree, py := amaTree(t, "test_show_message_offers_a_preview_only_before_anything_is_prepared") + event := amaEvent(t, tree) + // Rewind a copy of the Python fixture to the queued, unprepared state. The + // original remains intact for the after-send command. + before := t.TempDir() + raw, err := os.ReadFile(filepath.Join(tree, "state", "relay.sqlite3")) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(before, "relay.sqlite3"), raw, 0600); err != nil { + t.Fatal(err) + } + db, err := sql.Open("sqlite", filepath.Join(before, "relay.sqlite3")) + if err != nil { + t.Fatal(err) + } + for _, statement := range []string{"DELETE FROM attempt_messages", "DELETE FROM attempts", "UPDATE deliveries SET attempt_count = 0, state = 'queued', next_eligible_at = NULL"} { + if _, err := db.Exec(statement); err != nil { + t.Fatal(err) + } + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + home := pythonHome(t) + prior := golang(t, home, "--state", before, "show", "--event", event, "--message") + if prior.code != 0 { + t.Fatalf("before show: %+v", prior) + } + pre := decode(t, prior.stdout) + preview, present := pre["previewMessage"] + got := []any{pre["attemptMessages"], present, strings.HasSuffix(amaTokenCLI(t, preview), "-a1")} + payload := amaShow(t, tree, event) + entries := payload["attemptMessages"].([]any) + _, afterPreview := payload["previewMessage"] + got = append(got, afterPreview, eRequest(entries)) + amaCompare(t, got, py.Captures) +} +func eRequest(entries []any) any { return entries[0].(map[string]any)["requestId"] } +func Test21_AMA6_show_message_after_a_retry_lists_both_attempts_distinctly(t *testing.T) { + tree, py := amaTree(t, "test_show_message_after_a_retry_lists_both_attempts_distinctly") + payload := amaShow(t, tree, amaEvent(t, tree)) + entries := payload["attemptMessages"].([]any) + ids, statuses := []any{}, []any{} + for _, item := range entries { + e := item.(map[string]any) + ids = append(ids, e["requestId"]) + statuses = append(statuses, e["status"]) + } + got := []any{ids, statuses} + for _, item := range entries { + e := item.(map[string]any) + got = append(got, amaTokenCLI(t, e["message"])) + } + amaCompare(t, got, py.Captures) +} diff --git a/internal/relay/cli/registry.go b/internal/relay/cli/registry.go index 9f35896d..5d0208c4 100644 --- a/internal/relay/cli/registry.go +++ b/internal/relay/cli/registry.go @@ -6,9 +6,11 @@ import ( "flag" "fmt" "io" + "slices" "strings" "github.com/thisisjun786/codex-relay-workflow/internal/contract" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/delivery" "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" ) @@ -43,14 +45,7 @@ type Command struct { var Commands = []Command{doctorCommand, storeIdentityCommand, storeChallengeCommand, showCommand, statusCommand} // Registered reports whether this build implements the relay command name. -func Registered(name string) bool { - for _, command := range Commands { - if command.Name == name { - return true - } - } - return false -} +func Registered(name string) bool { return slices.Contains(allNames(), name) } // UsageError is SystemExit2: {"error": "usage", "detail": ...} with its own exit code. type UsageError struct { @@ -68,6 +63,9 @@ type PayloadExit struct { func (e *PayloadExit) Error() string { return fmt.Sprint(get(e.Payload, "detail")) } +// ExitPayload lets another relay package print this answer whole (delivery.PayloadError). +func (e *PayloadExit) ExitPayload() (contract.OrderedObject, int) { return e.Payload, e.Code } + // HostError is an unexpected failure whose Python class name is known, so the host envelope // can carry Python's f"{type(error).__name__}: {error}" unchanged. type HostError struct { @@ -120,6 +118,20 @@ func ExecuteAs(ctx context.Context, argv0 string, argv []string, stdout, stderr if len(remaining) == 0 { return parseError(globalUsage, "the following arguments are required: command") } + if slices.Contains(delivery.CommandNames(), remaining[0]) { + code, _ := delivery.ExecuteAs(ctx, prog, argv, stdout, stderr, func(selection store.StateSelection, socket string) error { + services := Services{Selection: selection, SocketPath: socket, AdapterRequested: socket != "", Program: program(argv0)} + refusal, err := selectionRefusal(services) + if err != nil { + return err + } + if refusal != nil { + return &PayloadExit{Payload: refusal, Code: contract.ExitRefused} + } + return nil + }) + return code + } var command *Command for i := range Commands { if Commands[i].Name == remaining[0] { @@ -255,18 +267,21 @@ func argparseMessage(err error) string { return text } -func commandNames() string { - names := make([]string, len(Commands)) - for i, command := range Commands { - names[i] = command.Name +// allNames is every relay command this build registers: this package's and delivery's. +func allNames() []string { + names := make([]string, 0, len(Commands)) + for _, command := range Commands { + names = append(names, command.Name) } - return strings.Join(names, ",") + return append(names, delivery.CommandNames()...) } +func commandNames() string { return strings.Join(allNames(), ",") } + func choices() string { - names := make([]string, len(Commands)) - for i, command := range Commands { - names[i] = store.PythonRepr(command.Name) + names := allNames() + for i, name := range names { + names[i] = store.PythonRepr(name) } return strings.Join(names, ", ") } diff --git a/internal/relay/delivery/ack.go b/internal/relay/delivery/ack.go new file mode 100644 index 00000000..02599a0b --- /dev/null +++ b/internal/relay/delivery/ack.go @@ -0,0 +1,848 @@ +package delivery + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "errors" + "fmt" + "math" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Acknowledgement refusals and words (ack.py). +const ( + WrongDeliveryKind = "wrong_delivery_kind" + AckProofMismatch = "ack_proof_mismatch" + AckTurnUnverified = "ack_turn_unverified" + NotAcknowledged = "not_acknowledged" + RestorationUndeliverable = "restoration_undeliverable" + DeliveryUnconfirmed = "delivery_unconfirmed" + AckPredatesAttempt = "ack_predates_attempt" +) + +var verdicts = []string{"verified", "needs_changes", "unverified", "aborted"} +var unconfirmed = []string{HeldUncertain, Sending} +var rejections = []string{"stale_generation", "unknown_generation", "duplicate_event", "relationship_not_active", "revision_mismatch"} + +var currencyReasons = map[string]string{StaleGeneration: StaleGeneration, SupersededRevision: SupersededRevision, RevisionAmbiguous: RevisionAmbiguous, RelationshipNotActive: RelationshipNotActive} + +// SyncHook is the verdict's sync outbox obligation (sync.enqueue_verdict_in), owned by todo 23. +// Nil writes none, as Python does when no target is configured. +type SyncHook func(ctx context.Context, relationship Relationship, event Row, verdict string, findings []any, record Obj, digest any, ruling int64) error + +// Ack is ack.AckService. +type Ack struct { + Store *store.Store + Delivery *Service + Clock Clock + Criteria *Criteria + Sync SyncHook +} + +func NewAck(d *Service) *Ack { + return &Ack{Store: d.Store, Delivery: d, Clock: d.Clock, Criteria: &Criteria{Store: d.Store, Clock: d.Clock}} +} + +func basis(row Row) [3]any { + return [3]any{row.S("state"), row.I("attempt_count"), row.Opt("dispatch_turn_id")} +} + +// Evaluate is evaluate: the rejection a parent should use, or "". +func (a *Ack) Evaluate(ctx context.Context, eventID string) (string, error) { + row, err := a.Delivery.Find(ctx, eventID) + if err != nil { + return "", err + } + if row != nil && row.S("kind") != Completion { + return "", refuse(WrongDeliveryKind, "%s is a %s, which is not acknowledged by a parent", store.PyRepr(eventID), row.S("kind")) + } + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil { + return "", err + } + if event == nil { + return "unknown_generation", nil + } + r, err := LoadRelationship(ctx, a.Store, event.S("relationship_id")) + if err != nil { + return "", err + } + if r.Status != "active" { + return "relationship_not_active", nil + } + if r.generation(event.I("execution_generation")) == nil { + return "unknown_generation", nil + } + if event.I("execution_generation") < r.Generation { + return "stale_generation", nil + } + existing, err := one(ctx, a.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return "", err + } + if existing != nil && existing.S("verified") == "verified" { + return "duplicate_event", nil + } + return "", nil +} + +// ClaimVerification is claim_verification: idempotent per event, reopened only by a moved set. +func (a *Ack) ClaimVerification(ctx context.Context, eventID string, turnID any) (string, error) { + now := a.Clock.ISO() + result := "already_claimed" + err := a.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + inserted, err := execSQL(ctx, a.Store, "INSERT OR IGNORE INTO verification_claims (event_id, claim_turn_id, claimed_at) VALUES (?,?,?)", eventID, turnID, now) + if err != nil { + return err + } + claimed, reclaimed := inserted == 1, false + if !claimed { + current, err := a.rulingIsCurrent(ctx, eventID) + if err != nil { + return err + } + if !current { + bound, err := a.Criteria.BoundDigest(ctx, eventID) + if err != nil { + return err + } + open, err := a.reReviewOpen(ctx, eventID, bound) + if err != nil { + return err + } + if open { + if _, err := execSQL(ctx, a.Store, "UPDATE verification_claims SET claim_turn_id = ?, claimed_at = ? WHERE event_id = ?", turnID, now, eventID); err != nil { + return err + } + if _, err := execSQL(ctx, a.Store, "DELETE FROM claim_context WHERE event_id = ?", eventID); err != nil { + return err + } + reclaimed = true + if err := journal(ctx, a.Store, "review_reclaimed", eventID, Obj{{Key: "claimTurnId", Value: turnID}}, now); err != nil { + return err + } + } + } + } + if claimed || reclaimed { + result = "proceed" + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil { + return err + } + if event != nil { + return a.Criteria.BindReview(ctx, event.S("relationship_id"), eventID) + } + } + return nil + }) + return result, err +} + +func (a *Ack) currentDigest(ctx context.Context, rid string) (any, error) { + registered, err := a.Criteria.Get(ctx, rid) + if err != nil || registered == nil { + return nil, err + } + return str(registered, "setDigest"), nil +} + +func (a *Ack) rulingIsCurrent(ctx context.Context, eventID string) (bool, error) { + row, err := one(ctx, a.Store, "SELECT set_digest FROM verdict_context WHERE event_id = ?", eventID) + if err != nil || row == nil { + return false, err + } + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil || event == nil { + return false, err + } + digest, err := a.currentDigest(ctx, event.S("relationship_id")) + return row.Opt("set_digest") == digest, err +} + +func (a *Ack) reReviewOpen(ctx context.Context, eventID string, decided any) (bool, error) { + settled, err := one(ctx, a.Store, "SELECT verdict FROM verdicts WHERE event_id = ?", eventID) + if err != nil { + return false, err + } + if settled != nil && settled.S("verdict") != "verified" { + return false, nil + } + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil || event == nil { + return false, err + } + digest, err := a.currentDigest(ctx, event.S("relationship_id")) + if err != nil || decided == digest { + return false, err + } + relationship, err := one(ctx, a.Store, "SELECT * FROM relationships WHERE relationship_id = ?", event.S("relationship_id")) + if err != nil || relationship == nil { + return false, err + } + state, err := Currency(ctx, a.Store, relationship, event) + if err != nil { + return false, err + } + current, _ := get(state, "current") + return current == true, nil +} + +// AckProof is identity.ack_proof. +func AckProof(eventID, turnID string) string { + sum := sha256.Sum256([]byte(eventID + "|" + turnID)) + return hex.EncodeToString(sum[:]) +} + +// Acknowledge is acknowledge. adapter nil records the parent's authored intent as unverified. +func (a *Ack) Acknowledge(ctx context.Context, eventID, ackTurn, proof string, accepted bool, rejection any, adapter Adapter) (Obj, error) { + row, err := a.Delivery.Find(ctx, eventID) + if err != nil { + return nil, err + } + if row == nil { + return nil, refuse(NotClaimable, "no delivery for %s", store.PyRepr(eventID)) + } + if row.S("kind") != Completion { + return nil, refuse(WrongDeliveryKind, "%s is a %s; contract v1 acknowledgements are parent-authored for a completion event", store.PyRepr(eventID), row.S("kind")) + } + existing, err := one(ctx, a.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return nil, err + } + if existing != nil && existing.S("verified") == "verified" { + return settledAck(existing), nil + } + state := row.S("state") + if !slices.Contains(append([]string{Dispatched, InboxOnly}, unconfirmed...), state) { + return nil, refuse(NotClaimable, "%s is %s; only a delivered event, or one whose send is still being confirmed, is acknowledged", store.PyRepr(eventID), store.PyRepr(state)) + } + if proof != AckProof(eventID, ackTurn) { + return nil, refuse(AckProofMismatch, "the proof does not match this event and turn; quoting the delivered fields back cannot produce it") + } + verification, err := a.verifyAckTurn(ctx, row, ackTurn, adapter) + if err != nil { + if !slices.Contains(unconfirmed, state) || Reason(err) != AckTurnUnverified { + return nil, err + } + verification = AckTurnUnverified + } + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil { + return nil, err + } + now := a.Clock.ISO() + var result Obj + err = a.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + already, err := one(ctx, a.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return err + } + if already != nil && already.S("verified") == "verified" { + result = settledAck(already) + return nil + } + fresh, err := a.Delivery.Find(ctx, eventID) + if err != nil { + return err + } + if fresh == nil || !slices.Contains(append([]string{Dispatched, InboxOnly, Acknowledged}, unconfirmed...), fresh.S("state")) { + what := "absent" + if fresh != nil { + what = fresh.S("state") + } + return refuse(NotClaimable, "%s became %s before this acknowledgement could be written", store.PyRepr(eventID), store.PyRepr(what)) + } + isUnconfirmed := slices.Contains(unconfirmed, fresh.S("state")) + kept := isUnconfirmed || basis(fresh) != basis(row) + why := "changed while its turn was read" + if isUnconfirmed { + why = fresh.S("state") + } + stored := verification + if kept { + stored = "unverified_turn" + } + computed, err := a.Evaluate(ctx, eventID) + if err != nil { + return err + } + if accepted && computed != "" { + return refuse(DispositionConflict, "this event cannot be accepted: %s", computed) + } + if !accepted && !truthy(rejection) { + rejection = computed + if computed == "" { + rejection = "revision_mismatch" + } + } + if accepted { + rejection = nil + } else if r, _ := rejection.(string); !slices.Contains(rejections, r) { + return refuse(DispositionConflict, "unknown rejection %s", pyReprValue(rejection)) + } + record := Obj{{Key: "eventId", Value: eventID}, {Key: "relationshipId", Value: event.S("relationship_id")}, {Key: "executionGeneration", Value: event.I("execution_generation")}, {Key: "revisionHash", Value: event.S("revision_hash")}, + {Key: "ackTurnId", Value: ackTurn}, {Key: "accepted", Value: accepted}, {Key: "rejectionReason", Value: rejection}, {Key: "ackAt", Value: now}, {Key: "ackProof", Value: proof}} + if _, err := execSQL(ctx, a.Store, "INSERT INTO acks (event_id, record, ack_turn_id, accepted, verified, rejection_reason, ack_at) VALUES (?,?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET record=excluded.record, ack_turn_id=excluded.ack_turn_id, accepted=excluded.accepted, verified=excluded.verified, rejection_reason=excluded.rejection_reason, ack_at=excluded.ack_at", + eventID, dumps(record), ackTurn, boolFlag(accepted), stored, rejection, now); err != nil { + return err + } + if stored == "verified" { + if _, err := execSQL(ctx, a.Store, "UPDATE deliveries SET state = ?, updated_at = ? WHERE event_id = ?", Acknowledged, now, eventID); err != nil { + return err + } + } + if kept { + err = a.writeAckEvidence(ctx, eventID, "unverified", fmt.Sprintf("kept as authored: the relay could not yet confirm this delivery for the turn it read (%s), and completes the acknowledgement once it does", why), DeliveryUnconfirmed, nil, nil, now, false) + } else if verification == "verified" { + err = a.writeAckEvidence(ctx, eventID, "host_read", nil, nil, nil, nil, now, false) + } else { + detail := "the host did not confirm this turn" + if adapter == nil { + detail = "no host adapter in this process" + } + err = a.writeAckEvidence(ctx, eventID, "unverified", detail, verification, nil, nil, now, false) + } + if err != nil { + return err + } + journalled := Obj{{Key: "accepted", Value: accepted}, {Key: "verified", Value: stored}, {Key: "reason", Value: rejection}} + if kept { + journalled = append(journalled, F{Key: "deliveryUnconfirmed", Value: why}) + } + if err := journal(ctx, a.Store, "acknowledged", eventID, journalled, now); err != nil { + return err + } + result = append(append(Obj(nil), record...), F{Key: "_verified", Value: stored}) + if kept { + result = append(result, F{Key: "_deliveryUnconfirmed", Value: why}) + } + return nil + }) + return result, err +} + +func settledAck(existing Row) Obj { + record := loadsObj(existing.S("record")) + return append(record, F{Key: "_verified", Value: existing.S("verified")}, F{Key: "_replay", Value: true}) +} + +// certainlyBefore is ack.certainly_before at the host's whole-second precision. +func certainlyBefore(started float64, sentAt string) bool { + sent, ok := epoch(sentAt) + return ok && started+TurnStartPrecisionSeconds <= sent +} + +func (a *Ack) verifyAckTurn(ctx context.Context, row Row, ackTurn string, adapter Adapter) (string, error) { + if adapter == nil { + return "unverified_turn", nil + } + attempt, err := one(ctx, a.Store, "SELECT * FROM attempts WHERE event_id = ? ORDER BY attempt_no DESC LIMIT 1", row.S("event_id")) + if err != nil { + return "", err + } + turn, err := adapter.ReadTurn(row.S("recipient_thread_id"), ackTurn) + if err != nil || turn == nil || turn.StartedAt == nil { + return "unverified_turn", nil + } + if attempt != nil { + sentAt := attempt.S("sent_at") + if attempt.N("sent_at") { + sentAt = attempt.S("observed_at") + } + var dispatched any + if !attempt.N("record") { + dispatched, _ = get(loadsObj(attempt.S("record")), "turnId") + } + if !truthy(dispatched) && attempt.S("affirmative_evidence") == TurnFound { + dispatched = row.Opt("dispatch_turn_id") + } + if ackTurn != dispatched && certainlyBefore(*turn.StartedAt, sentAt) { + return "", refuse(AckTurnUnverified, "turn %s started before the delivery, so it cannot be its acknowledgement", store.PyRepr(ackTurn)) + } + } + return "verified", nil +} + +func (a *Ack) writeAckEvidence(ctx context.Context, eventID, tier string, detail, reason, fingerprint, nextCheck any, now string, bump bool) error { + b := int64(0) + if bump { + b = 1 + } + _, err := execSQL(ctx, a.Store, "INSERT INTO ack_evidence (event_id, tier, detail, attempts, last_reason, fingerprint, next_check_at, observed_at) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET tier = excluded.tier, detail = excluded.detail, attempts = ack_evidence.attempts + ?, last_reason = excluded.last_reason, fingerprint = excluded.fingerprint, next_check_at = excluded.next_check_at, observed_at = excluded.observed_at", + eventID, tier, detail, b, reason, fingerprint, nextCheck, now, b) + return err +} + +func (a *Ack) ackEvidenceTier(ctx context.Context, eventID string) (string, error) { + row, err := one(ctx, a.Store, "SELECT tier FROM ack_evidence WHERE event_id = ?", eventID) + if err != nil || row == nil { + return "unrecorded", err + } + return row.S("tier"), nil +} + +func restorationResult(outcome, basis string, criterion any, detail string) Obj { + return Obj{{Key: "outcome", Value: outcome}, {Key: "basis", Value: basis}, {Key: "criterion", Value: criterion}, {Key: "detail", Value: detail}} +} + +func projectCap(findings []any) Obj { + for i, f := range findings { + o := f.(Obj) + if v, _ := get(o, "restoration"); truthy(v) { + where := fmt.Sprintf("finding %d of %d", i+1, len(findings)) + id, _ := get(o, "id") + if i < manifestLines { + return restorationResult("carried", "relay-message/legacy", id, fmt.Sprintf("%s, within the %d this renderer shows", where, manifestLines)) + } + return restorationResult("truncated", "relay-message/legacy", id, fmt.Sprintf("%s, past the %d this renderer shows", where, manifestLines)) + } + } + return restorationResult("not_carried", "relay-message/legacy", nil, "no finding declared a restoration block") +} + +// RecordVerdict is record_verdict: the verdict, the generation it opens and the correction, in +// one transaction, with currency decided inside it. No parameter bypasses the check. +func (a *Ack) RecordVerdict(ctx context.Context, eventID, verdict, verdictTurn string, criteria, findings []any, reason, expected any) (Obj, error) { + if !slices.Contains(verdicts, verdict) { + return nil, refuse(DispositionConflict, "unknown verdict %s", store.PyRepr(verdict)) + } + normalised, err := NormaliseFindings(criteria, findings) + if err != nil { + return nil, err + } + now := a.Clock.ISO() + var record Obj + err = a.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + settled, err := one(ctx, a.Store, "SELECT v.record AS record, c.set_digest AS set_digest FROM verdicts v LEFT JOIN verdict_context c ON c.event_id = v.event_id WHERE v.event_id = ?", eventID) + if err != nil { + return err + } + reReview := false + if settled != nil { + if reReview, err = a.reReviewOpen(ctx, eventID, settled.Opt("set_digest")); err != nil { + return err + } + } + if settled != nil && !reReview { + record = append(loadsObj(settled.S("record")), F{Key: "_replay", Value: true}) + return nil + } + if reReview && verdict != "verified" && verdict != "needs_changes" { + return refuse(DispositionConflict, "%s cannot replace the verified ruling this re-review is reopening: it would leave the assignment with no state to act on. Rule verified or needs_changes, or change the relationship's status", store.PyRepr(verdict)) + } + if reReview && expected == nil { + return refuse(CriteriaSetChanged, "%s was ruled against a different criteria set, so this is a re-review, and a re-review names the set it read: pass the reviewed digest explicitly", store.PyRepr(eventID)) + } + ack, err := one(ctx, a.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return err + } + if ack == nil || ack.I("accepted") == 0 || ack.S("verified") != "verified" { + return refuse(NotAcknowledged, "%s has no verified acceptance, so there is nothing to rule on", store.PyRepr(eventID)) + } + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil { + return err + } + relationship, err := RequireActive(ctx, a.Store, event.S("relationship_id")) + if err != nil { + return err + } + relRow, err := one(ctx, a.Store, "SELECT * FROM relationships WHERE relationship_id = ?", event.S("relationship_id")) + if err != nil { + return err + } + state, err := Currency(ctx, a.Store, relRow, event) + if err != nil { + return err + } + current, _ := get(state, "current") + if (verdict == "verified" || verdict == "needs_changes") && current != true { + return refuse(currencyReasons[str(state, "reason")], "%s cannot be ruled %s: %s", store.PyRepr(eventID), store.PyRepr(verdict), str(state, "detail")) + } + cover, err := a.Criteria.Coverage(ctx, event.S("relationship_id"), eventID, verdict, normalised, reason, expected) + if err != nil { + return err + } + var projected Obj + if verdict == "needs_changes" { + if !slices.Contains(relationship.AllowedRecipients, relationship.Child.TaskID) { + return refuse(RecipientNotAuthorized, "child %s is not an allowed recipient, so a revision cannot be routed to it", store.PyRepr(relationship.Child.TaskID)) + } + projected = projectCap(normalised) + if o := str(projected, "outcome"); o == "truncated" || o == "budget_dropped" { + return refuse(RestorationUndeliverable, "this correction declares a restoration block on %s that the revision message would not carry: %s. Move it within the first %d findings and rule again. No execution generation has been opened", pyReprValue(func() any { v, _ := get(projected, "criterion"); return v }()), str(projected, "detail"), manifestLines) + } + } else { + projected = restorationResult("not_carried", "relay-message/legacy", nil, fmt.Sprintf("a %s verdict opens no correction, so no message carries a restoration block", verdict)) + } + record = Obj{{Key: "eventId", Value: eventID}, {Key: "relationshipId", Value: event.S("relationship_id")}, {Key: "executionGeneration", Value: event.I("execution_generation")}, {Key: "verdict", Value: verdict}, {Key: "verdictTurnId", Value: verdictTurn}, {Key: "decidedAt", Value: now}} + if len(normalised) > 0 { + record = append(record, F{Key: "criteria", Value: normalised}) + } + var next any + correction := "" + if verdict == "needs_changes" { + rid, child := relationship.ID, relationship.Child.TaskID + revisionEvent, err := store.RevisionRequestEventID(rid, eventID, verdictTurn) + if err != nil { + return err + } + number, err := OpenGenerationIn(ctx, a.Store, a.Clock, rid, "revision-"+revisionEvent, "needs_changes_revision", nil) + if err != nil { + return err + } + criteriaList := normalised + if criteriaList == nil { + criteriaList = []any{} + } + payload := Obj{{Key: "eventId", Value: revisionEvent}, {Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: number}, {Key: "kind", Value: Revision}, {Key: "supersedesEvent", Value: eventID}, {Key: "supersedesRevisionHash", Value: event.S("revision_hash")}, + {Key: "verdict", Value: verdict}, {Key: "verdictTurnId", Value: verdictTurn}, {Key: "criteria", Value: criteriaList}, {Key: "childTaskId", Value: child}, {Key: "emittedAt", Value: now}, + {Key: "note", Value: "relay-owned revision request; contract v1 defines no record for this direction"}} + if _, err := execSQL(ctx, a.Store, "INSERT OR IGNORE INTO events (event_id, relationship_id, execution_generation, revision_hash, outcome, producer, attempt, turn_thread_id, turn_id, turn_status, receipt, stage, first_seen_at, last_seen_at, observation_count) VALUES (?,?,?,?,?,?,NULL,?,?,?,?, 'final', ?,?,1)", + revisionEvent, rid, number, store.NoDeliverable, "revision_request", "relay", relationship.Parent.TaskID, verdictTurn, "completed", dumps(payload), now, now); err != nil { + return err + } + if err := a.Delivery.EnqueueIn(ctx, revisionEvent, rid, Revision, child); err != nil { + return err + } + record = append(record, F{Key: "nextExecutionGeneration", Value: number}) + next, correction = number, revisionEvent + } + if _, err := execSQL(ctx, a.Store, "INSERT INTO verdicts (event_id, record, verdict, next_generation, verdict_turn_id, decided_at) VALUES (?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET record = excluded.record, verdict = excluded.verdict, next_generation = excluded.next_generation, verdict_turn_id = excluded.verdict_turn_id, decided_at = excluded.decided_at", + eventID, dumps(record), verdict, next, verdictTurn, now); err != nil { + return err + } + if err := journal(ctx, a.Store, "verdict_recorded", eventID, Obj{{Key: "verdict", Value: verdict}}, now); err != nil { + return err + } + if err := journal(ctx, a.Store, "restoration_projected", eventID, projected, now); err != nil { + return err + } + if correction != "" { + if err := journal(ctx, a.Store, "restoration_projected", correction, projected, now); err != nil { + return err + } + } + setDigest, _ := get(cover, "setDigest") + if reReview { + if err := journal(ctx, a.Store, "verdict_superseded", eventID, Obj{{Key: "supersededVerdict", Value: loadsObj(settled.S("record"))}, {Key: "reviewedSetDigest", Value: settled.Opt("set_digest")}, {Key: "currentSetDigest", Value: setDigest}}, now); err != nil { + return err + } + } + var findingsText any + if len(normalised) > 0 { + findingsText = dumps(normalised) + } + currency := "current" + if current != true { + currency = str(state, "reason") + if currency == "" { + currency = "unknown" + } + } + tier, err := a.ackEvidenceTier(ctx, eventID) + if err != nil { + return err + } + headID, _ := get(state, "headEventId") + headRev, _ := get(state, "headRevisionHash") + if _, err := execSQL(ctx, a.Store, "INSERT INTO verdict_context (event_id, set_digest, coverage, findings, reason, currency, head_event_id, head_revision, ack_evidence, recorded_at) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET set_digest = excluded.set_digest, coverage = excluded.coverage, findings = excluded.findings, reason = excluded.reason, currency = excluded.currency, head_event_id = excluded.head_event_id, head_revision = excluded.head_revision, ack_evidence = excluded.ack_evidence, recorded_at = excluded.recorded_at", + eventID, setDigest, str(cover, "coverage"), findingsText, reason, currency, headID, headRev, tier, now); err != nil { + return err + } + if a.Sync != nil { + seen, err := one(ctx, a.Store, "SELECT COUNT(*) AS seen FROM journal WHERE kind = ? AND subject = ?", "verdict_superseded", eventID) + if err != nil { + return err + } + return a.Sync(ctx, relationship, event, verdict, normalised, record, setDigest, 1+seen.I("seen")) + } + return nil + }) + return record, err +} + +// OpenGenerationIn is registry.open_generation_in inside the caller's transaction. +func OpenGenerationIn(ctx context.Context, s *store.Store, clock Clock, rid, dispatchRequest, reason string, dispatchTurn any) (int64, error) { + replay, err := one(ctx, s, "SELECT execution_generation FROM generations WHERE relationship_id = ? AND dispatch_request_id = ?", rid, dispatchRequest) + if err != nil { + return 0, err + } + if replay != nil { + return replay.I("execution_generation"), nil + } + current, err := one(ctx, s, "SELECT execution_generation, status, superseded_by FROM relationships WHERE relationship_id = ?", rid) + if err != nil { + return 0, err + } + if current == nil { + return 0, refuse(UnregisteredRelationship, "no relationship %s", store.PyRepr(rid)) + } + if current.S("status") != "active" || truthy(current.Opt("superseded_by")) { + return 0, refuse(RelationshipNotActive, "relationship %s is not active", store.PyRepr(rid)) + } + number := current.I("execution_generation") + 1 + now := clock.ISO() + anchor, bound := "anchor_pending", any(nil) + if dispatchTurn != nil { + anchor, bound = "bound", now + } + if _, err := execSQL(ctx, s, "INSERT INTO generations (relationship_id, execution_generation, dispatch_request_id, anchor_state, dispatch_turn_id, reason, opened_at, bound_at) VALUES (?,?,?,?,?,?,?,?)", rid, number, dispatchRequest, anchor, dispatchTurn, reason, now, bound); err != nil { + return 0, err + } + if _, err := execSQL(ctx, s, "UPDATE relationships SET execution_generation = ?, updated_at = ? WHERE relationship_id = ?", number, now, rid); err != nil { + return 0, err + } + if err := journal(ctx, s, "generation_opened", rid, Obj{{Key: "generation", Value: number}, {Key: "reason", Value: reason}}, now); err != nil { + return 0, err + } + _, err = execSQL(ctx, s, "INSERT INTO delivery_supersession (event_id, reason, noted_at, applied) SELECT d.event_id, 'stale_generation', ?, 0 FROM deliveries d JOIN events e ON e.event_id = d.event_id WHERE d.relationship_id = ? AND e.execution_generation < ? AND e.outcome NOT IN ('merge_turn_grant') AND d.state IN ('queued','sending','held_uncertain','dispatched','deferred_busy','withheld_pre_send','inbox_only') ON CONFLICT(event_id) DO NOTHING", now, rid, number) + return number, err +} + +// RestorationOf is restoration_of. +func (a *Ack) RestorationOf(ctx context.Context, eventID string) (Obj, error) { + row, err := one(ctx, a.Store, "SELECT detail FROM journal WHERE kind = ? AND subject = ? ORDER BY seq DESC LIMIT 1", "restoration_projected", eventID) + if err != nil { + return nil, err + } + if row == nil || row.S("detail") == "" { + return Obj{{Key: "outcome", Value: "unmeasured"}, {Key: "basis", Value: nil}, {Key: "criterion", Value: nil}, {Key: "detail", Value: "this ruling was recorded before the relay measured restoration delivery"}}, nil + } + return loadsObj(row.S("detail")), nil +} + +func pendingBackoff(attemptNo int64) float64 { + return math.Min(900, 30*math.Pow(2, float64(max(0, attemptNo-1)))) +} + +func (a *Ack) pendingFingerprint(ctx context.Context, eventID string, delivery Row) (string, error) { + event, err := a.Delivery.eventRow(ctx, eventID) + if err != nil { + return "", err + } + var rel Row + if event != nil { + if rel, err = one(ctx, a.Store, "SELECT status, execution_generation FROM relationships WHERE relationship_id = ?", event.S("relationship_id")); err != nil { + return "", err + } + } + parts := []string{"absent", "absent", "0", "0"} + if delivery != nil { + parts[0] = delivery.S("state") + } + if rel != nil { + parts[1], parts[2] = rel.S("status"), fmt.Sprint(rel.I("execution_generation")) + } + if event != nil { + parts[3] = fmt.Sprint(event.I("execution_generation")) + } + sum := sha256.Sum256([]byte(strings.Join(parts, "|"))) + return hex.EncodeToString(sum[:]), nil +} + +// VerifyPendingAcks is verify_pending_acks: complete acknowledgements authored without a host, +// re-checking disposition as acknowledge does. +func (a *Ack) VerifyPendingAcks(ctx context.Context, adapter Adapter, limit int, nowp *float64) ([]any, error) { + now := a.Clock.Now() + if nowp != nil { + now = *nowp + } + if limit == 0 { + limit = 8 + } + pending, err := all(ctx, a.Store, "SELECT a.event_id, a.ack_turn_id, a.ack_at, a.accepted, COALESCE(e.attempts, 0) AS attempts, e.last_reason, e.fingerprint, e.next_check_at FROM acks a LEFT JOIN ack_evidence e ON e.event_id = a.event_id LEFT JOIN deliveries d ON d.event_id = a.event_id WHERE a.verified = 'unverified_turn' AND (e.next_check_at IS NULL OR e.next_check_at <= ? OR (e.last_reason = ? AND d.state IN (?, ?))) ORDER BY COALESCE(e.next_check_at, 0), a.event_id LIMIT ?", + now, DeliveryUnconfirmed, Dispatched, InboxOnly, limit) + if err != nil { + return nil, err + } + results := []any{} + for _, p := range pending { + eventID := p.S("event_id") + row, err := a.Delivery.Find(ctx, eventID) + if err != nil { + return nil, err + } + if row == nil { + continue + } + verification, err := a.verifyAckTurn(ctx, row, p.S("ack_turn_id"), adapter) + if err != nil { + if Reason(err) == "" { + return nil, err + } + verification = Reason(err) + } + result, err := a.settlePendingAck(ctx, eventID, p, verification, now, row) + if err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil +} + +func (a *Ack) settlePendingAck(ctx context.Context, eventID string, pending Row, verification string, now float64, basisRow Row) (Obj, error) { + stamp := a.Clock.ISO() + var out Obj + err := a.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + fresh, err := one(ctx, a.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return err + } + if fresh == nil || fresh.S("verified") == "verified" { + out = Obj{{Key: "eventId", Value: eventID}, {Key: "outcome", Value: "already_settled"}} + return nil + } + if fresh.S("ack_turn_id") != pending.S("ack_turn_id") || fresh.S("ack_at") != pending.S("ack_at") { + out = Obj{{Key: "eventId", Value: eventID}, {Key: "outcome", Value: "replaced"}} + return nil + } + delivery, err := a.Delivery.Find(ctx, eventID) + if err != nil { + return err + } + if basisRow != nil && delivery != nil && basis(delivery) != basis(basisRow) { + out = Obj{{Key: "eventId", Value: eventID}, {Key: "outcome", Value: "changed"}} + return nil + } + blocker := "" + switch { + case delivery != nil && slices.Contains(unconfirmed, delivery.S("state")): + blocker = DeliveryUnconfirmed + case delivery == nil || !slices.Contains([]string{Dispatched, InboxOnly, Acknowledged}, delivery.S("state")): + blocker = "delivery_state_changed" + default: + computed, err := a.Evaluate(ctx, eventID) + if Reason(err) != "" { + computed = Reason(err) + } else if err != nil { + return err + } + if fresh.I("accepted") != 0 && computed != "" { + blocker = computed + } + } + if blocker == "" && delivery != nil { + current, err := one(ctx, a.Store, "SELECT sent_at FROM attempts WHERE event_id = ? AND attempt_no = ?", eventID, delivery.I("attempt_count")) + if err != nil { + return err + } + if current != nil && !current.N("sent_at") && fresh.S("ack_at") < current.S("sent_at") { + blocker = AckPredatesAttempt + } + } + fingerprint, err := a.pendingFingerprint(ctx, eventID, delivery) + if err != nil { + return err + } + if blocker == "" && verification == "verified" { + if _, err := execSQL(ctx, a.Store, "UPDATE acks SET verified = 'verified' WHERE event_id = ?", eventID); err != nil { + return err + } + if _, err := execSQL(ctx, a.Store, "UPDATE deliveries SET state = ?, updated_at = ? WHERE event_id = ?", Acknowledged, stamp, eventID); err != nil { + return err + } + if err := a.writeAckEvidence(ctx, eventID, "host_read", nil, nil, fingerprint, nil, stamp, true); err != nil { + return err + } + out = Obj{{Key: "eventId", Value: eventID}, {Key: "outcome", Value: "verified"}} + return journal(ctx, a.Store, "ack_verified", eventID, Obj{{Key: "tier", Value: "host_read"}}, stamp) + } + reason := blocker + if reason == "" { + reason = verification + } + unchanged := pending.Opt("last_reason") == reason && pending.Opt("fingerprint") == fingerprint + if err := a.writeAckEvidence(ctx, eventID, "unverified", "the acknowledgement stands exactly as authored; it was not promoted", reason, fingerprint, now+pendingBackoff(pending.I("attempts")+1), stamp, true); err != nil { + return err + } + out = Obj{{Key: "eventId", Value: eventID}, {Key: "outcome", Value: "withheld"}, {Key: "reason", Value: reason}} + if !unchanged { + return journal(ctx, a.Store, "ack_verification_withheld", eventID, Obj{{Key: "reason", Value: reason}}, stamp) + } + return nil + }) + return out, err +} + +// BindDispatchedRevision is bind_dispatched_revision. +func (a *Ack) BindDispatchedRevision(ctx context.Context, revisionEvent string) (Obj, error) { + row, err := a.Delivery.Find(ctx, revisionEvent) + if err != nil || row == nil || row.S("state") != Dispatched || !truthy(row.Opt("dispatch_turn_id")) { + return nil, err + } + event, err := a.Delivery.eventRow(ctx, revisionEvent) + if err != nil { + return nil, err + } + return BindAnchor(ctx, a.Store, a.Clock, event.S("relationship_id"), event.I("execution_generation"), row.S("dispatch_turn_id")) +} + +// BindAnchor is registry.bind_anchor from a dispatch receipt: idempotent, never rebinding. +func BindAnchor(ctx context.Context, s *store.Store, clock Clock, rid string, number int64, turn string) (Obj, error) { + if strings.TrimSpace(turn) == "" { + return nil, refuse("unbound_generation", "an anchor needs an exact dispatch turn id") + } + r, err := LoadRelationship(ctx, s, rid) + if err != nil { + return nil, err + } + current := r.generation(number) + if current == nil { + return nil, refuse(UnknownGeneration, "%s has no generation %d", store.PyRepr(rid), number) + } + if current.S("anchor_state") == "bound" { + if current.S("dispatch_turn_id") == turn { + return generationRecord(current), nil + } + return nil, refuse("anchor_already_bound", "generation %d is already bound to %s", number, pyReprValue(current.Opt("dispatch_turn_id"))) + } + now := clock.ISO() + err = s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, s, "UPDATE generations SET anchor_state = ?, dispatch_turn_id = ?, bound_at = ? WHERE relationship_id = ? AND execution_generation = ?", "bound", turn, now, rid, number); err != nil { + return err + } + return journal(ctx, s, "anchor_bound", rid, Obj{{Key: "generation", Value: number}}, now) + }) + if err != nil { + return nil, err + } + g, err := one(ctx, s, "SELECT * FROM generations WHERE relationship_id = ? AND execution_generation = ?", rid, number) + return generationRecord(g), err +} + +func generationRecord(g Row) Obj { + return Obj{{Key: "relationshipId", Value: g.S("relationship_id")}, {Key: "executionGeneration", Value: g.I("execution_generation")}, {Key: "dispatchRequestId", Value: g.S("dispatch_request_id")}, {Key: "anchorState", Value: g.S("anchor_state")}, + {Key: "dispatchTurnId", Value: g.Opt("dispatch_turn_id")}, {Key: "openedAt", Value: g.S("opened_at")}, {Key: "boundAt", Value: g.Opt("bound_at")}, {Key: "reason", Value: g.Opt("reason")}} +} + +// BindPendingAnchors is bind_pending_anchors: recovery over state, never moving a bound anchor. +func (a *Ack) BindPendingAnchors(ctx context.Context) ([]any, error) { + rows, err := all(ctx, a.Store, "SELECT d.event_id FROM deliveries d JOIN events e ON e.event_id = d.event_id JOIN generations g ON g.relationship_id = e.relationship_id AND g.execution_generation = e.execution_generation WHERE d.kind = ? AND d.state IN (?,?) AND d.dispatch_turn_id IS NOT NULL AND g.anchor_state = ? ORDER BY d.updated_at LIMIT ?", Revision, Dispatched, Acknowledged, "anchor_pending", 50) + if err != nil { + return nil, err + } + bound := []any{} + for _, r := range rows { + result, err := a.BindDispatchedRevision(ctx, r.S("event_id")) + var refused *Refused + if errors.As(err, &refused) { + continue + } + if err != nil { + return nil, err + } + if result != nil { + bound = append(bound, r.S("event_id")) + } + } + return bound, nil +} diff --git a/internal/relay/delivery/ackreconcile_a_test.go b/internal/relay/delivery/ackreconcile_a_test.go new file mode 100644 index 00000000..6a3a09d4 --- /dev/null +++ b/internal/relay/delivery/ackreconcile_a_test.go @@ -0,0 +1,198 @@ +package delivery + +import ( + "strings" + "testing" + "time" +) + +// test_ack_reconcile.py ACR-1..ACR-23. Each Test21_ACR runs the Go twin of every Python test the +// property lists (a subtest per Python test), in the Python test's own tree, and compares every +// asserted value, the delivery tables and the sends with what Python produced +// (hostloss_harness_test.go mirror + testdata/capture.py). + +const acr = "test_ack_reconcile" + +// noArchiveInfo is the Python tests' NoArchiveInfo: is_archived answers None, the rest passes through. +type noArchiveInfo struct{ Adapter } + +func (noArchiveInfo) IsArchived(string, any) (*bool, error) { return nil, nil } + +// sentAt is datetime.fromtimestamp(1789420929.360483, timezone.utc).isoformat(). +func sentAt() string { + return time.UnixMicro(1789420929360483).UTC().Format("2006-01-02T15:04:05.000000-07:00") +} + +// ackDispatched is Acknowledgement._dispatched. +func (h *hl) ackDispatched() (string, TurnInfo) { + event := h.queuedEvent(regOpts{}) + h.attemptOn(event, h.host, nil) + h.clock.Advance(5) + return event, h.host.startTurn(parent, "parent-ack-turn", "inProgress", "") +} + +// refusal records the reason a Python assertRefused compared (it records caught.exception.reason). +func (h *hl) refusal(_ any, err error) { + h.t.Helper() + if err == nil { + h.t.Fatal("expected a refusal") + } + h.eq(Reason(err)) +} + +func Test21_ACR01_turn_start_precision(t *testing.T) { + mirror(t, acr, "TurnStartPrecision.test_a_turn_starting_in_the_same_second_as_its_send_is_not_refused", func(h *hl) { + h.eq(certainlyBefore(1789420929, sentAt())) + }) + mirror(t, acr, "TurnStartPrecision.test_a_genuinely_earlier_whole_second_turn_is_still_refused", func(h *hl) { + h.eq(certainlyBefore(1789420920, sentAt())) + h.eq(certainlyBefore(1789420928, sentAt())) + }) + mirror(t, acr, "TurnStartPrecision.test_a_later_turn_is_never_refused", func(h *hl) { + h.eq(certainlyBefore(1789420930, sentAt())) + }) +} + +func Test21_ACR02_archive_observation(t *testing.T) { + mirror(t, acr, "UnknownArchiveState.test_an_unknown_archive_observation_is_not_evidence_of_being_unarchived", func(h *hl) { + o := Observe(noArchiveInfo{h.host}, parent, nil, true) + h.eq(o.Deliverable) + h.eq(o.WithholdReason) + h.eq(o.MaySend()) + }) + mirror(t, acr, "UnknownArchiveState.test_a_confirmed_unarchived_recipient_stays_deliverable", func(h *hl) { + f := false + h.host.threads[parent].archived = &f + h.eq(Observe(h.host, parent, nil, true).MaySend()) + }) + mirror(t, acr, "UnknownArchiveState.test_a_confirmed_archived_recipient_stays_blocked", func(h *hl) { + yes := true + h.host.threads[parent].archived = &yes + o := Observe(h.host, parent, nil, true) + h.eq(o.MaySend()) + h.eq(o.WithholdReason) + }) + mirror(t, acr, "UnknownArchiveState.test_relaxing_the_evidence_requirement_is_explicit", func(h *hl) { + h.eq(Observe(noArchiveInfo{h.host}, parent, nil, false).MaySend()) + }) +} + +func Test21_ACR03_a_correct_proof_from_a_real_later_turn_closes_the_attempt(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_a_correct_proof_from_a_real_later_turn_closes_the_attempt", func(h *hl) { + event, turn := h.ackDispatched() + result, err := h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host) + mustDo(t, err) + h.eq(field(result, "accepted")) + h.eq(field(result, "rejectionReason")) + h.eq(field(result, "_verified")) + h.eq(h.row(event).S("state")) + }) +} + +func Test21_ACR04_an_ack_proof_mismatch_is_refused(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_an_echo_without_the_proof_never_closes_the_attempt", func(h *hl) { + event, turn := h.ackDispatched() + h.refusal(h.ack.Acknowledge(h.ctx, event, turn.TurnID, strings.Repeat("0", 64), true, nil, h.host)) + h.eq(h.row(event).S("state")) + }) + mirror(t, acr, "Acknowledgement.test_a_proof_computed_over_someone_else_turn_is_refused", func(h *hl) { + event, turn := h.ackDispatched() + h.refusal(h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, "a-different-turn"), true, nil, h.host)) + }) +} + +func Test21_ACR05_an_unknown_turn_is_kept_unverified(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_a_turn_that_does_not_exist_does_not_close_the_attempt", func(h *hl) { + event, _ := h.ackDispatched() + result, err := h.ack.Acknowledge(h.ctx, event, "invented-turn", AckProof(event, "invented-turn"), true, nil, h.host) + mustDo(t, err) + h.eq(field(result, "_verified")) + h.eq(h.row(event).S("state")) + }) +} + +func Test21_ACR06_a_turn_that_started_before_the_delivery_is_refused(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_a_turn_that_started_before_the_delivery_is_refused", func(h *hl) { + event := h.queuedEvent(regOpts{}) + old := h.host.startTurn(parent, "older-turn", "completed", "") + h.clock.Advance(120) + h.attemptOn(event, h.host, at(h.clock.Now())) + h.refusal(h.ack.Acknowledge(h.ctx, event, old.TurnID, AckProof(event, old.TurnID), true, nil, h.host)) + }) +} + +func Test21_ACR07_an_advanced_generation_is_a_disposition_conflict(t *testing.T) { + t.Run("advanced before the ack", func(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_a_generation_that_advanced_after_dispatch_cannot_be_accepted", func(h *hl) { + event, turn := h.ackDispatched() + h.openGeneration("later", "needs_changes_revision", "later-turn") + h.refusal(h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host)) + }) + }) + t.Run("advanced during the ack", func(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_a_generation_advancing_during_the_acknowledgement_is_caught", func(h *hl) { + event, turn := h.ackDispatched() + racy := actsDuringTheTurnRead(h.host, func() { h.openGeneration("racy", "needs_changes_revision", "racy-turn") }) + // Python replaces read_turn permanently; the action here runs on the first read, the + // only one the acknowledgement makes. + h.refusal(h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, racy)) + h.eq(h.one("SELECT * FROM acks WHERE event_id = ?", event)) + }) + }) +} + +func Test21_ACR08_one_event_is_claimed_for_verification_once(t *testing.T) { + mirror(t, acr, "Acknowledgement.test_one_event_is_verified_only_once", func(h *hl) { + event := h.queuedEvent(regOpts{}) + first, err := h.ack.ClaimVerification(h.ctx, event, "t1") + mustDo(t, err) + h.eq(first) + second, err := h.ack.ClaimVerification(h.ctx, event, "t2") + mustDo(t, err) + h.eq(second) + }) + mirror(t, acr, "Acknowledgement.test_a_duplicate_delivery_cannot_cause_a_second_verification", func(h *hl) { + event := h.queuedEvent(regOpts{}) + first, err := h.ack.ClaimVerification(h.ctx, event, "t1") + mustDo(t, err) + second, err := h.ack.ClaimVerification(h.ctx, event, "t1") + mustDo(t, err) + h.eq(first, second) + h.eq(h.count("SELECT COUNT(*) AS c FROM verification_claims")) + }) +} + +// verdictAcknowledged is Verdicts._acknowledged (and VerdictAtomicity._acknowledged). +func (h *hl) verdictAcknowledged(recipients []string) string { + event := h.queuedEvent(regOpts{recipients: recipients}) + h.attemptOn(event, h.host, nil) + h.clock.Advance(5) + turn := h.host.startTurn(parent, "ack-turn", "inProgress", "") + _, err := h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host) + mustDo(h.t, err) + return event +} + +func (h *hl) verdict(event, verdict, turn string) (Obj, error) { + return h.ack.RecordVerdict(h.ctx, event, verdict, turn, nil, nil, nil, nil) +} + +func Test21_ACR09_a_verdict_requires_a_verified_acceptance(t *testing.T) { + mirror(t, acr, "Verdicts.test_a_verdict_requires_a_verified_acceptance", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.attemptOn(event, h.host, nil) + h.refusal(h.verdict(event, "verified", "v1")) + }) +} + +func Test21_ACR10_needs_changes_opens_a_generation_and_queues_a_revision(t *testing.T) { + mirror(t, acr, "Verdicts.test_needs_changes_opens_a_generation_and_queues_a_revision_to_the_same_child", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + record, err := h.verdict(event, "needs_changes", "verdict-1") + mustDo(t, err) + h.eq(field(record, "nextExecutionGeneration")) + revision := h.one("SELECT * FROM deliveries WHERE kind = ?", Revision) + h.eq(revision != nil) + h.eq(revision.S("recipient_task_id")) + }) +} diff --git a/internal/relay/delivery/ackreconcile_b_test.go b/internal/relay/delivery/ackreconcile_b_test.go new file mode 100644 index 00000000..b4b70ebb --- /dev/null +++ b/internal/relay/delivery/ackreconcile_b_test.go @@ -0,0 +1,216 @@ +package delivery + +import ( + "fmt" + "slices" + "strings" + "testing" +) + +// test_ack_reconcile.py ACR-11..ACR-20 (see ackreconcile_a_test.go). + +func Test21_ACR11_a_revision_cannot_be_routed_to_an_unauthorized_child(t *testing.T) { + mirror(t, acr, "Verdicts.test_a_revision_cannot_be_routed_to_an_unauthorized_child", func(h *hl) { + event := h.verdictAcknowledged([]string{parent}) + h.refusal(h.verdict(event, "needs_changes", "verdict-1")) + }) +} + +func Test21_ACR12_a_revision_request_is_never_acknowledged_by_the_parent_path(t *testing.T) { + mirror(t, acr, "Verdicts.test_a_revision_request_is_never_acknowledged_by_the_parent_path", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + _, err := h.verdict(event, "needs_changes", "verdict-1") + mustDo(t, err) + revision := h.one("SELECT * FROM deliveries WHERE kind = ?", Revision) + h.refusal(h.ack.Evaluate(h.ctx, revision.S("event_id"))) + }) +} + +func Test21_ACR13_a_dispatched_revision_binds_the_new_generation_anchor(t *testing.T) { + mirror(t, acr, "Verdicts.test_a_dispatched_revision_binds_the_new_generation_anchor", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + _, err := h.verdict(event, "needs_changes", "verdict-1") + mustDo(t, err) + revision := h.one("SELECT * FROM deliveries WHERE kind = ?", Revision) + h.attemptOn(revision.S("event_id"), h.host, at(h.clock.Now())) + bound, err := h.ack.BindDispatchedRevision(h.ctx, revision.S("event_id")) + mustDo(t, err) + h.eq(field(bound, "anchorState")) + h.eq(truthy(field(bound, "dispatchTurnId"))) + }) +} + +// uncertain is Reconciliation._uncertain. +func (h *hl) uncertain(outcome string) (string, string) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{outcome} + record := h.attemptOn(event, h.host, nil) + h.eq(str(record, "deliveryState")) + return event, str(record, "requestId") +} + +func Test21_ACR14_the_operation_receipt_is_read_before_the_recipient_turns(t *testing.T) { + mirror(t, acr, "Reconciliation.test_the_operation_receipt_is_checked_before_the_recipient_turns", func(h *hl) { + _, request := h.uncertain("turn_start_fail") + var order []any + ordered := &hooked{Adapter: h.host, + getOperation: func(id string) (Obj, error) { order = append(order, "operation"); return h.host.GetOperation(id) }, + findToken: func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + order = append(order, "scan") + return h.host.FindToken(thread, token, limit, messageOnly) + }} + h.reconcile(request, ordered) + h.eq(order[:2]) + }) +} + +func Test21_ACR15_no_affirmative_evidence_keeps_the_attempt_held(t *testing.T) { + mirror(t, acr, "Reconciliation.test_no_affirmative_evidence_keeps_the_attempt_held_and_says_what_is_missing", func(h *hl) { + _, request := h.uncertain("turn_start_fail") + outcome := h.reconcile(request, h.host) + h.eq(str(outcome, "evidence")) + h.eq(str(outcome, "state")) + h.eq(pyIn("no confirmed pre-send rejection", field(outcome, "missing"))) + h.eq(pyIn("exhausted", field(outcome, "recipientScan"))) + }) + mirror(t, acr, "Reconciliation.test_elapsed_time_never_changes_the_outcome", func(h *hl) { + event, request := h.uncertain("turn_start_fail") + first := h.reconcile(request, h.host) + h.clock.Advance(86400 * 30) + second := h.reconcile(request, h.host) + h.eq(str(first, "evidence")) + h.eq(str(second, "state")) + h.eq(len(h.attemptsFor(event))) + }) +} + +func Test21_ACR16_a_token_in_the_recipient_items_advances_the_delivery_honestly(t *testing.T) { + mirror(t, acr, "Reconciliation.test_a_token_found_in_the_recipient_items_advances_the_delivery_honestly", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.host.startTurn(parent, "", "completed", "..."+request+"...") + outcome := h.reconcile(request, h.host) + h.eq(str(outcome, "evidence")) + h.eq(h.row(event).S("state")) + h.eq(h.row(event).S("dispatch_evidence")) + stored := loadsObj(h.attemptsFor(event)[0].S("record")) + h.eq(field(stored, "transportReceiptStatus")) + h.eq(field(stored, "sendAttempted")) + h.eq(field(stored, "deliveryState")) + h.eq(field(sub(stored, "reconciliation"), "affirmativeEvidence")) + }) +} + +func Test21_ACR17_a_truncated_scan_is_inconclusive(t *testing.T) { + mirror(t, acr, "Reconciliation.test_a_truncated_scan_is_inconclusive_rather_than_absent", func(h *hl) { + _, request := h.uncertain("turn_start_fail") + for i := 0; i < 30; i++ { + h.host.startTurn(parent, "", "completed", fmt.Sprintf("noise %d", i)) + } + h.host.scanLimit = 5 + outcome := h.reconcile(request, h.host) + h.eq(str(outcome, "evidence")) + h.eq(strings.Contains(str(outcome, "recipientScan"), "exhausted=False")) + }) +} + +func Test21_ACR18_a_missing_ledger_row_is_an_observation_not_a_licence_to_resend(t *testing.T) { + mirror(t, acr, "Reconciliation.test_a_missing_ledger_row_is_an_observation_not_a_licence_to_resend", func(h *hl) { + _, request := h.uncertain("turn_start_fail") + delete(h.host.ledger, request) + outcome := h.reconcile(request, h.host) + h.eq(str(outcome, "operationObservation")) + h.eq(str(outcome, "evidence")) + h.clock.Advance(86400) + eligible := []any{} + for _, e := range h.eligible() { + eligible = append(eligible, e) + } + h.eq(eligible) + }) +} + +func Test21_ACR19_a_confirmed_pre_send_rejection_permits_a_new_attempt(t *testing.T) { + mirror(t, acr, "Reconciliation.test_a_confirmed_pre_send_rejection_permits_a_new_attempt", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.host.ledger[request] = preSendRejection(request) + outcome := h.reconcile(request, h.host) + h.eq(str(outcome, "evidence")) + h.clock.Advance(100000) + second := h.attemptOn(event, h.host, at(h.clock.Now())) + h.eq(field(second, "attemptNo")) + }) +} + +func (h *hl) recoverOnStart() Obj { + h.t.Helper() + report, err := h.rc.RecoverOnStart(h.ctx, h.host, nil) + mustDo(h.t, err) + return report +} + +func listed(report Obj, key string, value any) bool { + list, _ := field(report, key).([]any) + return slices.Contains(list, value) +} + +func Test21_ACR20_restart_recovery_never_sends(t *testing.T) { + t.Run("crash before the ledger row", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_a_crash_before_the_transport_ledger_row_recovers_without_resending", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"process_death"} + record := h.attemptOn(event, h.host, nil) + h.eq(str(record, "deliveryState")) + delete(h.host.ledger, str(record, "requestId")) + report := h.recoverOnStart() + h.eq(listed(report, "heldUncertain", str(record, "requestId"))) + h.eq(field(report, "resent")) + }) + }) + t.Run("crash during the send", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_a_crash_during_the_send_recovers_as_uncertain", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + record := h.attemptOn(event, h.host, nil) + report := h.recoverOnStart() + h.eq(listed(report, "heldUncertain", str(record, "requestId"))) + }) + }) + t.Run("crash after acceptance", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_a_crash_after_acceptance_recovers_from_the_ledger", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + turn := h.host.startTurn(parent, "", "inProgress", "") + h.host.ledger[request] = Obj{{Key: "requestId", Value: request}, {Key: "status", Value: "accepted"}, {Key: "resumed", Value: Obj{{Key: "approvalPolicy", Value: "never"}}}, {Key: "turnId", Value: turn.TurnID}} + report := h.recoverOnStart() + h.eq(h.row(event).S("state")) + h.eq(field(report, "heldUncertain")) + }) + }) + t.Run("app restart", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_an_app_restart_recovers_pending_work_without_duplicating_it", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + h.attemptOn(event, h.host, nil) + h.host.readFailures = map[string]bool{} // FakeHostAdapter.restart + h.recoverOnStart() + h.eq(len(h.host.sends)) + h.eq(len(h.attemptsFor(event))) + }) + }) +} + +// pyIn is Python's `needle in haystack` over a list (membership) or a string (substring). +func pyIn(needle string, haystack any) bool { + switch v := haystack.(type) { + case []any: + return slices.Contains(v, any(needle)) + case string: + return strings.Contains(v, needle) + } + return false +} diff --git a/internal/relay/delivery/ackreconcile_c_test.go b/internal/relay/delivery/ackreconcile_c_test.go new file mode 100644 index 00000000..9834bc76 --- /dev/null +++ b/internal/relay/delivery/ackreconcile_c_test.go @@ -0,0 +1,228 @@ +package delivery + +import ( + "context" + "sort" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_ack_reconcile.py ACR-21..ACR-23 (see ackreconcile_a_test.go). + +func Test21_ACR21_recovery_reports_awaiting_acks_and_never_a_correction(t *testing.T) { + t.Run("dispatched completion", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_a_dispatched_delivery_awaiting_acknowledgement_is_reported_not_resent", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.attemptOn(event, h.host, nil) + report := h.recoverOnStart() + h.eq(field(report, "awaitingAck")) + h.eq(field(report, "resent")) + }) + }) + t.Run("dispatched correction", func(t *testing.T) { + mirror(t, acr, "RestartRecovery.test_a_dispatched_correction_is_not_reported_as_awaiting_acknowledgement", func(h *hl) { + first := h.verdictAcknowledged([]string{parent, child}) + _, err := h.ack.RecordVerdict(h.ctx, first, "needs_changes", "v1", nil, []any{Obj{{Key: "id", Value: "tie"}, {Key: "verdict", Value: "needs_changes"}, {Key: "note", Value: "equal timestamps"}}}, nil, nil) + mustDo(t, err) + correction := h.one("SELECT event_id FROM deliveries WHERE kind = ?", Revision).S("event_id") + h.attemptOn(correction, h.host, nil) + + h.clock.Advance(5) + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(t, err) + _, err = BindAnchor(h.ctx, h.store, h.clock, h.rid, r.Generation, "revision-turn") + mustDo(t, err) + turn := turnRef{child, "revision-turn", "completed"} + payload := h.readyPayload(h.rid, r.Generation, []string{h.artifact("revised.txt", "the corrected deliverable")}, 1, turn) + _, err = h.accept(payload, storeAcceptNone) + mustDo(t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(t, err) + h.attemptOn(str(payload, "eventId"), h.host, nil) + + report := h.recoverOnStart() + h.eq(field(report, "awaitingAck")) + h.eq(listed(report, "awaitingAck", correction)) + h.eq(field(report, "resent")) + h.eq(h.row(correction).S("state")) + h.eq(h.row(str(payload, "eventId")).S("state")) + h.eq(len(h.host.sends)) + }) + }) +} + +// counts is VerdictAtomicity._counts. +func (h *hl) counts() []any { + return []any{ + h.count("SELECT COUNT(*) AS c FROM generations"), + h.count("SELECT COUNT(*) AS c FROM verdicts"), + h.count("SELECT COUNT(*) AS c FROM deliveries WHERE kind = 'revision_request'"), + } +} + +// failQueueing makes the correction's delivery insert fail inside the verdict transaction: the +// storage failure the Python test injects by replacing delivery.enqueue_in. A TEMP trigger lives +// on the store's one connection only and never touches the file's schema. +func (h *hl) failQueueing() (restore func()) { + h.t.Helper() + _, err := h.store.DB.ExecContext(h.ctx, "CREATE TEMP TRIGGER fail_queueing BEFORE INSERT ON main.deliveries WHEN NEW.kind = 'revision_request' BEGIN SELECT RAISE(ABORT, 'storage failed while queueing the correction'); END") + mustDo(h.t, err) + return func() { + _, err := h.store.DB.ExecContext(h.ctx, "DROP TRIGGER temp.fail_queueing") + mustDo(h.t, err) + } +} + +// failAtCommit makes COMMIT itself fail (store.fault_hook raising): the verdict insert leaves a +// dangling deferred foreign key that SQLite checks only at COMMIT. +func (h *hl) failAtCommit() (restore func()) { + h.t.Helper() + for _, stmt := range []string{ + "CREATE TEMP TABLE commit_fault_parent (id INTEGER PRIMARY KEY)", + "CREATE TEMP TABLE commit_fault_child (id INTEGER PRIMARY KEY, parent INTEGER REFERENCES commit_fault_parent(id) DEFERRABLE INITIALLY DEFERRED)", + "CREATE TEMP TRIGGER fail_at_commit AFTER INSERT ON main.verdicts BEGIN INSERT INTO commit_fault_child (parent) VALUES (-1); END", + } { + _, err := h.store.DB.ExecContext(h.ctx, stmt) + mustDo(h.t, err) + } + return func() { + for _, stmt := range []string{"DROP TRIGGER temp.fail_at_commit", "DROP TABLE temp.commit_fault_child", "DROP TABLE temp.commit_fault_parent"} { + _, err := h.store.DB.ExecContext(h.ctx, stmt) + mustDo(h.t, err) + } + } +} + +func (h *hl) mustFail(_ any, err error) { + h.t.Helper() + if err == nil { + h.t.Fatal("the injected storage fault did not fail the verdict") + } +} + +func Test21_ACR22_a_needs_changes_verdict_is_atomic(t *testing.T) { + t.Run("normal path", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_the_normal_path_produces_a_generation_a_verdict_and_a_revision", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + _, err := h.verdict(event, "needs_changes", "v1") + mustDo(t, err) + h.eq(h.counts()) + }) + }) + t.Run("fault while queueing", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_a_storage_failure_while_queueing_the_correction_rolls_everything_back", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + before := h.counts() + restore := h.failQueueing() + h.mustFail(h.verdict(event, "needs_changes", "v1")) + restore() + h.eq(h.counts()) + h.eq(before[0]) + }) + }) + t.Run("fault then retry", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_a_storage_failure_leaves_the_verdict_retryable", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + restore := h.failQueueing() + h.mustFail(h.verdict(event, "needs_changes", "v1")) + restore() + record, err := h.verdict(event, "needs_changes", "v1") + mustDo(t, err) + h.eq(field(record, "nextExecutionGeneration")) + h.eq(h.counts()) + }) + }) + t.Run("fault at commit", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_a_fault_at_commit_time_also_rolls_back", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + restore := h.failAtCommit() + h.mustFail(h.verdict(event, "needs_changes", "v1")) + restore() + h.eq(h.counts()) + }) + }) +} + +// contractOnly is the test's _contract: the verdict without the relay's internal markers. +func contractOnly(record Obj) Obj { + out := Obj{} + for _, f := range record { + if len(f.Key) == 0 || f.Key[0] != '_' { + out = append(out, f) + } + } + return out +} + +func Test21_ACR23_a_replayed_verdict_allocates_nothing_further(t *testing.T) { + t.Run("sequential replay", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_a_replayed_verdict_allocates_nothing_further", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + first, err := h.verdict(event, "needs_changes", "v1") + mustDo(t, err) + again, err := h.verdict(event, "needs_changes", "v1") + mustDo(t, err) + requireSameJSON(t, "replayed contract", contractOnly(again), jsonable(contractOnly(first))) + h.eq(contractOnly(first)) + h.eq(truthy(field(first, "_replay"))) + h.eq(truthy(field(again, "_replay"))) + h.eq(h.counts()) + }) + }) + t.Run("two concurrent writers", func(t *testing.T) { + mirror(t, acr, "VerdictAtomicity.test_concurrent_replay_allocates_one_generation_and_one_revision", func(h *hl) { + event := h.verdictAcknowledged([]string{parent, child}) + var ( + mu sync.Mutex + results []Obj + errs = []any{} + ready sync.WaitGroup + start = make(chan struct{}) + done sync.WaitGroup + ) + for i := 0; i < 2; i++ { + ready.Add(1) + done.Add(1) + go func() { + defer done.Done() + s, err := store.Open(context.Background(), h.store.Path, "") + if err != nil { + ready.Done() + mu.Lock() + errs = append(errs, err.Error()) + mu.Unlock() + return + } + defer func() { _ = s.Close() }() + ack := NewAck(NewService(s, h.clock)) + ready.Done() + <-start + record, err := ack.RecordVerdict(context.Background(), event, "needs_changes", "v1", nil, nil, nil, nil) + mu.Lock() + defer mu.Unlock() + if err != nil { + errs = append(errs, err.Error()) + return + } + results = append(results, record) + }() + } + ready.Wait() + close(start) + done.Wait() + h.eq(errs) + h.eq(len(results)) + if len(results) != 2 { + t.Fatalf("both writers should succeed: %v", errs) + } + requireSameJSON(t, "the two contract records", contractOnly(results[1]), jsonable(contractOnly(results[0]))) + h.eq(contractOnly(results[0])) + replays := []bool{truthy(field(results[0], "_replay")), truthy(field(results[1], "_replay"))} + sort.Slice(replays, func(i, j int) bool { return !replays[i] && replays[j] }) + h.eq([]any{replays[0], replays[1]}) + h.eq(h.counts()) + }) + }) +} diff --git a/internal/relay/delivery/adapter.go b/internal/relay/delivery/adapter.go new file mode 100644 index 00000000..4d9c302f --- /dev/null +++ b/internal/relay/delivery/adapter.go @@ -0,0 +1,57 @@ +package delivery + +// HostError is a host read or send that could not complete. Kind is the Python exception class +// name a message about it carries (ConnectionError for the fake host's failed reads). +type HostError struct { + Kind string + Message string +} + +func (e *HostError) Error() string { return e.Message } + +func errorLabel(err error) string { + if h, ok := err.(*HostError); ok { + return h.Kind + ": " + h.Message + } + return "Exception: " + err.Error() +} + +// ThreadFacts is hostadapter.ThreadFacts. +type ThreadFacts struct { + RuntimeStatus string + CanAcceptInput *bool +} + +// TurnInfo is hostadapter.TurnInfo. +type TurnInfo struct { + TurnID string + Status string + StartedAt *float64 +} + +// TokenScan is hostadapter.TokenScan. +type TokenScan struct { + Found bool + TurnID any + Exhausted bool + Scanned int + OtherTurn, OtherKind any +} + +// Adapter is hostadapter.HostAdapter: reads, and the one supported send. Nothing here can change +// a task's model, effort, sandbox, approval policy, goal or archive state. +type Adapter interface { + ReadThread(thread string) (ThreadFacts, error) + IsArchived(thread string, cwd any) (*bool, error) + ReadGoalStatus(thread string) (any, error) + ListTurnIDs(thread string, limit int) ([]string, error) + ReadTurn(thread, turn string) (*TurnInfo, error) + SendMessage(requestID, thread, message string, settings *TaskSettings) (Obj, error) + GetOperation(requestID string) (Obj, error) + FindToken(thread, token string, limit int, messageOnly bool) (TokenScan, error) + FindDispatchedTurn(thread, turnID string, sentAt float64) (TurnPresence, error) + FindTokenSince(thread, token string, older []string, limit int) (TokenScan, error) + FindTokenInTurn(thread, token, turnID string, limit int) (TokenScan, error) + // RecipientFingerprint digests the newest items' content, so an append shows up. + RecipientFingerprint(thread string) (string, error) +} diff --git a/internal/relay/delivery/anchor_test.go b/internal/relay/delivery/anchor_test.go new file mode 100644 index 00000000..76581389 --- /dev/null +++ b/internal/relay/delivery/anchor_test.go @@ -0,0 +1,268 @@ +package delivery + +import ( + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_anchor_binding.py ANB-1..ANB-8. The daemon tick is todo 29; its binding contract is +// ported here as AnchorPasses (both passes, counted and not assigned, around reconciliation). + +type anb struct { + *vcu + rc *Reconciler +} + +func newANB(t *testing.T, tree string) *anb { + v := newVCU(t, tree) + return &anb{v, NewReconciler(v.delivery)} +} + +func (a *anb) revisionPending() string { + e := a.queuedEvent(regOpts{recipients: []string{parent, child}}) + a.mustAttempt(e, nil) + a.clock.Advance(5) + turn := a.host.startTurn(parent, "ack-turn", "inProgress", "") + _, err := a.ack.Acknowledge(a.ctx, e, turn.TurnID, AckProof(e, turn.TurnID), true, nil, a.host) + mustDo(a.t, err) + _, err = a.ack.RecordVerdict(a.ctx, e, "needs_changes", "verdict-1", nil, nil, nil, nil) + mustDo(a.t, err) + return a.one("SELECT event_id FROM deliveries WHERE kind = ?", Revision).S("event_id") +} + +func (a *anb) gen2() Obj { + return generationRecord(a.one("SELECT * FROM generations WHERE relationship_id = ? AND execution_generation = 2", a.rid)) +} + +func (a *anb) childReceipt() map[string]any { + anchor := a.one("SELECT dispatch_turn_id FROM generations WHERE relationship_id = ? AND execution_generation = 2", a.rid).S("dispatch_turn_id") + if anchor == "" { + anchor = "turn-unbound" + } + payload := a.readyPayload(a.rid, 2, []string{a.artifact("fixed.txt", "the correction")}, 1, turnRef{child, anchor, "completed"}) + stored, err := a.accept(payload, store.AcceptOptions{}) + if err != nil { + return refusalOf(err) + } + record := loadsObj(stored.Record) + return map[string]any{"ok": append(record, F{Key: "_duplicate", Value: stored.Duplicate}, F{Key: "_pathBindingMode", Value: stored.PathBinding.String}, F{Key: "_stage", Value: stored.Stage})} +} + +func (a *anb) lostSettleWrite() (string, string) { + rev := a.revisionPending() + a.clock.Advance(3600) + a.host.script = []string{"in_progress"} + record := a.mustAttempt(rev, at(a.clock.Now())) + turn := a.host.startTurn(child, "", "inProgress", "") + request := str(record, "requestId") + a.host.ledger[request] = Obj{{Key: "requestId", Value: request}, {Key: "status", Value: "accepted"}, {Key: "resumed", Value: Obj{{Key: "approvalPolicy", Value: "never"}}}, {Key: "turnId", Value: turn.TurnID}} + return rev, turn.TurnID +} + +func (a *anb) recover() Obj { + out, err := a.rc.RecoverOnStart(a.ctx, a.host, nil) + mustDo(a.t, err) + return out +} + +func (a *anb) bindPending() []any { + bound, err := a.ack.BindPendingAnchors(a.ctx) + mustDo(a.t, err) + return bound +} + +func runANB(t *testing.T, mode string, goSide func(a *anb, out map[string]any)) { + tree := t.TempDir() + python := runPython(t, tree, "anb", mode) + a := newANB(t, tree) + out := map[string]any{} + goSide(a, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + requireSameTables(t, a.fixture, python) +} + +func (a *anb) dispatchRevision(script string) string { + rev := a.revisionPending() + a.clock.Advance(3600) + if script != "" { + a.host.script = []string{script} + } + return rev +} + +func TestANB01_every_route_to_dispatched_binds_the_new_anchor(t *testing.T) { + t.Run("a later tick", func(t *testing.T) { + runANB(t, "tick", func(a *anb, out map[string]any) { + rev := a.dispatchRevision("") + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["pending"] = str(a.gen2(), "anchorState") + report := &AnchorReport{} + reconciled := []any{} + mustDo(t, AnchorPasses(a.ctx, a.ack, report, func() error { + r := a.recover() + v, _ := get(r, "reconciled") + reconciled = v.([]any) + return nil + })) + out["bound"] = []any{report.Passes[0], reconciled, report.Passes[1]} + out["gen2"] = a.gen2() + out["receipt"] = a.childReceipt() + }) + }) + t.Run("the recovery helper", func(t *testing.T) { + runANB(t, "recovery", func(a *anb, out map[string]any) { + rev := a.dispatchRevision("") + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["pending"] = str(a.gen2(), "anchorState") + out["bound"], out["again"] = a.bindPending(), a.bindPending() + out["gen2"] = a.gen2() + out["receipt"] = a.childReceipt() + }) + }) + t.Run("a reconcile promotion", func(t *testing.T) { + runANB(t, "reconcile", func(a *anb, out map[string]any) { + rev := a.dispatchRevision("transport_unknown") + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["promoted"] = a.recover() + out["bound"] = a.bindPending() + out["gen2"] = a.gen2() + out["state"] = a.row(rev).S("state") + }) + }) + t.Run("inside the promoting transaction", func(t *testing.T) { + runANB(t, "promotion", func(a *anb, out map[string]any) { + rev, turn := a.lostSettleWrite() + out["recovered"] = a.recover() + out["gen2"] = a.gen2() + if a.row(rev).S("state") != Dispatched || str(out["gen2"].(Obj), "dispatchTurnId") != turn { + t.Fatal("bound in the promotion") + } + out["receipt"] = a.childReceipt() + }) + }) +} + +func TestANB02_an_unbound_generation_refuses_the_childs_receipt(t *testing.T) { + runANB(t, "unbound", func(a *anb, out map[string]any) { + rev := a.dispatchRevision("") + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["pending"] = str(a.gen2(), "anchorState") + out["receipt"] = a.childReceipt() + if out["receipt"].(map[string]any)["reason"] != "unbound_generation" { + t.Fatal("unbound_generation") + } + }) +} + +func TestANB03_a_disagreeing_promotion_is_recorded_not_swallowed(t *testing.T) { + runANB(t, "conflict", func(a *anb, out map[string]any) { + a.lostSettleWrite() + _, err := BindAnchor(a.ctx, a.store, a.clock, a.rid, 2, "a-different-turn") + mustDo(t, err) + out["recovered"] = a.recover() + out["gen2"] = a.gen2() + if a.one("SELECT detail FROM journal WHERE kind = 'anchor_conflict'") == nil { + t.Fatal("the disagreement is journalled") + } + }) +} + +func TestANB04_a_stale_reconciliation_binds_nothing(t *testing.T) { + runANB(t, "stale", func(a *anb, out map[string]any) { + rev, _ := a.lostSettleWrite() + _, err := execSQL(a.ctx, a.store, "UPDATE deliveries SET attempt_count = attempt_count + 1 WHERE event_id = ?", rev) + mustDo(t, err) + // Python forces the snapshot check (_is_current) to True; the guarded UPDATE decides. + forceCurrent = true + defer func() { forceCurrent = false }() + out["recovered"] = a.recover() + out["gen2"] = a.gen2() + if str(out["gen2"].(Obj), "anchorState") != "anchor_pending" { + t.Fatal("a stale attempt bound the generation") + } + }) +} + +func TestANB05_a_revision_request_is_retired_once_the_child_answers(t *testing.T) { + for _, mode := range []string{"retired_ready", "retired_failed", "retired_daemon"} { + t.Run(mode, func(t *testing.T) { + runANB(t, mode, func(a *anb, out map[string]any) { + rev := a.dispatchRevision("") + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["pending"] = str(a.gen2(), "anchorState") + a.bindPending() + anchor := str(a.gen2(), "dispatchTurnId") + switch mode { + case "retired_ready": + out["receipt"] = a.childReceipt() + case "retired_failed": + payload := a.executionPayload(a.rid, 2, "failed", 1, turnRef{child, anchor, "completed"}) + stored, err := a.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + out["receipt"] = map[string]any{"ok": append(loadsObj(stored.Record), F{Key: "_duplicate", Value: false}, F{Key: "_pathBindingMode", Value: nil}, F{Key: "_stage", Value: stored.Stage})} + default: + a.host.startTurn(child, anchor, "failed", "") + stored, err := a.intake.DaemonObservation(a.ctx, a.rid, store.TurnReference{ThreadID: child, TurnID: anchor, Status: "failed"}) + mustDo(t, err) + out["receipt"] = append(loadsObj(stored.Record), F{Key: "_duplicate", Value: false}, F{Key: "_pathBindingMode", Value: nil}, F{Key: "_stage", Value: stored.Stage}) + } + reason, err := a.delivery.SupersessionReason(a.ctx, rev) + mustDo(t, err) + out["reason"] = reason + if reason != SupersededRevision { + t.Fatal("superseded_revision") + } + }) + }) + } +} + +func TestANB06_binding_is_idempotent_and_needs_a_dispatch(t *testing.T) { + for _, mode := range []string{"idempotent", "never"} { + t.Run(mode, func(t *testing.T) { + runANB(t, mode, func(a *anb, out map[string]any) { + script := "" + if mode == "never" { + script = "busy" + } + rev := a.dispatchRevision(script) + out["record"] = a.mustAttempt(rev, at(a.clock.Now())) + out["pending"] = str(a.gen2(), "anchorState") + out["bound"], out["again"] = a.bindPending(), a.bindPending() + out["gen2"] = a.gen2() + }) + }) + } +} + +func TestANB07_a_revision_promoted_during_a_tick_binds_in_that_tick(t *testing.T) { + var order []string + report := &AnchorReport{} + f := newANB(t, "") + mustDo(t, AnchorPasses(f.ctx, anchorBinderFunc(func() ([]any, error) { order = append(order, "bind"); return nil, nil }), report, func() error { + order = append(order, "reconcile") + return nil + })) + if len(order) != 3 || order[0] != "bind" || order[1] != "reconcile" || order[2] != "bind" { + t.Fatalf("order %v", order) + } +} + +func TestANB08_both_binding_passes_are_counted(t *testing.T) { + calls := 0 + report := &AnchorReport{} + f := newANB(t, "") + mustDo(t, AnchorPasses(f.ctx, anchorBinderFunc(func() ([]any, error) { + calls++ + if calls == 1 { + return []any{"a", "b"}, nil + } + return []any{}, nil + }), report, func() error { return nil })) + if report.AnchorsBound != 2 { + t.Fatalf("anchorsBound %d", report.AnchorsBound) + } +} diff --git a/internal/relay/delivery/anchors.go b/internal/relay/delivery/anchors.go new file mode 100644 index 00000000..d2b85e46 --- /dev/null +++ b/internal/relay/delivery/anchors.go @@ -0,0 +1,41 @@ +package delivery + +import "context" + +// AnchorBinder is the repair pass a tick runs (Ack.BindPendingAnchors). +type AnchorBinder interface { + BindPendingAnchors(ctx context.Context) ([]any, error) +} + +type anchorBinderFunc func() ([]any, error) + +func (f anchorBinderFunc) BindPendingAnchors(context.Context) ([]any, error) { return f() } + +// AnchorReport is the binding half of the daemon's TickReport. +type AnchorReport struct { + AnchorsBound int + Passes [][]any + Notes []string +} + +// AnchorPasses is the binding contract of RelayDaemon.tick (todo 29 owns the rest of the tick): +// bind, reconcile, bind again, so a revision promoted by reconciliation binds in the same tick, +// and the two passes are ADDED, so the second cannot erase what the first repaired. A failed +// pass is a note, never the end of the tick. +func AnchorPasses(ctx context.Context, binder AnchorBinder, report *AnchorReport, reconcile func() error) error { + pass := func() { + bound, err := binder.BindPendingAnchors(ctx) + if err != nil { + report.Notes = append(report.Notes, "anchor recovery failed: "+err.Error()) + return + } + report.AnchorsBound += len(bound) + report.Passes = append(report.Passes, bound) + } + pass() + if err := reconcile(); err != nil { + return err + } + pass() + return nil +} diff --git a/internal/relay/delivery/attempt_message_atomicity_test.go b/internal/relay/delivery/attempt_message_atomicity_test.go new file mode 100644 index 00000000..79f2f046 --- /dev/null +++ b/internal/relay/delivery/attempt_message_atomicity_test.go @@ -0,0 +1,164 @@ +package delivery + +import ( + "strings" + "testing" +) + +const ama = "test_attempt_message_atomicity" +const amaClass = "AttemptMessageAtomicity." + +type interleavedHost struct { + Adapter + before func() + fired bool +} + +func (h *interleavedHost) ListTurnIDs(thread string, limit int) ([]string, error) { + ids, err := h.Adapter.ListTurnIDs(thread, limit) + if !h.fired { + h.fired = true + h.before() + } + return ids, err +} + +func amaToken(t *testing.T, message string) string { + t.Helper() + for _, line := range strings.Split(message, "\n") { + if strings.HasPrefix(line, "requestId: ") { + return strings.TrimPrefix(line, "requestId: ") + } + } + t.Fatalf("message carries no requestId: %q", message) + return "" +} + +func amaMessages(h *hl, event string) []any { + h.t.Helper() + rows, err := h.delivery.AttemptMessages(h.ctx, event) + mustDo(h.t, err) + return rows +} +func amaSent(h *hl, request string) any { + h.t.Helper() + row, err := one(h.ctx, h.store, "SELECT message FROM attempt_messages WHERE request_id = ?", request) + mustDo(h.t, err) + if row == nil { + return nil + } + return row.Opt("message") +} + +func Test21_AMA1_interleaved_earlier_caller_cannot_shift_the_token(t *testing.T) { + mirror(t, ama, amaClass+"test_interleaved_earlier_caller_cannot_shift_the_token", func(h *hl) { + event := h.queuedEvent(regOpts{}) + earlier := NewService(h.store, h.clock) + adapter := &interleavedHost{Adapter: h.host} + adapter.before = func() { + h.host.script = []string{"busy"} + now := h.clock.Now() - 60 + _, err := earlier.Attempt(h.ctx, event, h.host, &now, "earlier-slow-caller") + mustDo(t, err) + } + record, err := h.delivery.Attempt(h.ctx, event, adapter, nil, "") + mustDo(t, err) + h.eq(adapter.fired) + h.eq(record != nil) + sent := h.host.sends[len(h.host.sends)-1] + h.eq(amaToken(t, sent.message)) + h.eq(str(record, "requestId")) + for _, s := range h.host.sends { + h.eq(amaToken(t, s.message)) + } + }) +} + +func Test21_AMA2_inspection_after_send_reports_the_sent_attempt_not_the_next_one(t *testing.T) { + mirror(t, ama, amaClass+"test_inspection_after_send_reports_the_sent_attempt_not_the_next_one", func(h *hl) { + event := h.queuedEvent(regOpts{}) + record := h.mustAttempt(event, nil) + rows := amaMessages(h, event) + h.eq(len(rows)) + entry := rows[0].(Obj) + h.eq(str(entry, "requestId")) + h.eq(str(entry, "status")) + h.eq(amaToken(t, str(entry, "message"))) + preview, err := h.delivery.PreviewMessage(h.ctx, event) + mustDo(t, err) + h.eq(amaToken(t, preview)) + _ = record + }) +} + +func Test21_AMA3_lost_response_reconciliation_searches_the_token_that_was_sent(t *testing.T) { + mirror(t, ama, amaClass+"test_lost_response_reconciliation_searches_the_token_that_was_sent", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"transport_unknown"} + record := h.mustAttempt(event, nil) + request := str(record, "requestId") + message := h.host.sends[len(h.host.sends)-1].message + h.eq(amaToken(t, message)) + h.eq(amaSent(h, request)) + h.eq(str(record, "deliveryState")) + h.host.startTurn(parent, "", "completed", message) + resolved := h.reconcile(request, h.host) + h.eq(str(resolved, "state")) + h.eq(str(resolved, "evidence")) + h.eq(strings.Contains(message, request)) + }) +} + +func Test21_AMA4_show_message_returns_the_sent_attempt_after_a_send(t *testing.T) { + mirror(t, ama, amaClass+"test_show_message_returns_the_sent_attempt_after_a_send", func(h *hl) { + event := h.queuedEvent(regOpts{}) + record := h.mustAttempt(event, nil) + entries := amaMessages(h, event) + h.eq(len(entries)) + h.eq(str(entries[0].(Obj), "requestId")) + h.eq(str(entries[0].(Obj), "status")) + h.eq(amaToken(t, str(entries[0].(Obj), "message"))) + h.eq(len(entries) == 0) // cmd_show previewMessage absence is exercised at the CLI surface. + _ = record + }) +} + +func Test21_AMA5_show_message_offers_a_preview_only_before_anything_is_prepared(t *testing.T) { + mirror(t, ama, amaClass+"test_show_message_offers_a_preview_only_before_anything_is_prepared", func(h *hl) { + event := h.queuedEvent(regOpts{}) + rows := amaMessages(h, event) + h.eq(rows) + preview, err := h.delivery.PreviewMessage(h.ctx, event) + mustDo(t, err) + h.eq(len(rows) == 0) + h.eq(strings.HasSuffix(amaToken(t, preview), "-a1")) + record := h.mustAttempt(event, nil) + after := amaMessages(h, event) + h.eq(len(after) == 0) + h.eq(str(after[0].(Obj), "requestId")) + _ = record + }) +} + +func Test21_AMA6_show_message_after_a_retry_lists_both_attempts_distinctly(t *testing.T) { + mirror(t, ama, amaClass+"test_show_message_after_a_retry_lists_both_attempts_distinctly", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"busy"} + h.mustAttempt(event, nil) + now := h.clock.Now() + 10000 + h.mustAttempt(event, &now) + entries := amaMessages(h, event) + ids, statuses := []any{}, []any{} + for _, v := range entries { + e := v.(Obj) + ids = append(ids, str(e, "requestId")) + statuses = append(statuses, str(e, "status")) + } + h.eq(ids) + h.eq(statuses) + for _, v := range entries { + e := v.(Obj) + h.eq(amaToken(t, str(e, "message"))) + } + }) +} diff --git a/internal/relay/delivery/bridge_reads.go b/internal/relay/delivery/bridge_reads.go new file mode 100644 index 00000000..7a95c06b --- /dev/null +++ b/internal/relay/delivery/bridge_reads.go @@ -0,0 +1,206 @@ +package delivery + +import ( + "encoding/json" + "strings" +) + +// BridgeReads is the recipient-reading half of bridge_adapter.BridgeHostAdapter: the four reads +// the host-loss and reconciliation rules depend on, over one call(method, params) -> result, the +// only way the adapter reaches the host. The adapter itself (transport, ledger, sends) is todo +// 28's; it embeds these so the paging, bounds and item typing stay the ones tested here. +type BridgeReads struct { + Call func(method string, params Obj) (Obj, error) + Page int +} + +// DispatchedTurnMaxPages is hostadapter.DISPATCHED_TURN_MAX_PAGES. +const DispatchedTurnMaxPages = 20 + +// BridgePage is bridge_adapter.PAGE. +const BridgePage = 50 + +func (b BridgeReads) page() int { + if b.Page > 0 { + return b.Page + } + return BridgePage +} + +func list(result Obj) []any { + data, _ := get(result, "data") + items, _ := data.([]any) + return items +} + +func cursorOf(result Obj) any { + next, _ := get(result, "nextCursor") + if !truthy(next) { + return nil + } + return next +} + +func number(v any) *float64 { + switch n := v.(type) { + case float64: + return &n + case int64: + f := float64(n) + return &f + case json.Number: + f, err := n.Float64() + if err == nil { + return &f + } + } + return nil +} + +// FindDispatchedTurn is find_dispatched_turn: newest first, without items, stopping at the first +// turn begun before the send. +func (b BridgeReads) FindDispatchedTurn(thread, turnID string, sentAt float64) (TurnPresence, error) { + var cursor any + read := func() (ListingPage, error) { + params := Obj{{Key: "threadId", Value: thread}, {Key: "limit", Value: int64(b.page())}, {Key: "itemsView", Value: "notLoaded"}, {Key: "sortDirection", Value: "desc"}} + if cursor != nil { + params = append(params, F{Key: "cursor", Value: cursor}) + } + result, err := b.Call("thread/turns/list", params) + if err != nil { + return ListingPage{}, err + } + cursor = cursorOf(result) + var turns []TurnInfo + for _, raw := range list(result) { + turn, _ := raw.(Obj) + id, _ := get(turn, "id") + status, ok := get(turn, "status") + if !ok { + status = "unknown" + } + started, _ := get(turn, "startedAt") + turns = append(turns, TurnInfo{TurnID: pyStrOrEmpty(id), Status: pyStr(status), StartedAt: number(started)}) + } + return ListingPage{Turns: turns, Follows: cursor != nil}, nil + } + return FindInListingPaged(read, DispatchedTurnMaxPages, turnID, sentAt) +} + +// itemText is bridge_adapter._item_text. +func itemText(entry Obj) string { + raw, _ := get(entry, "item") + item, ok := raw.(Obj) + if !ok { + return dumpsSorted(entry) + } + for _, key := range []string{"text", "preview", "summary", "aggregatedOutput"} { + if v, ok := get(item, key); ok { + if s, isText := v.(string); isText { + return s + } + } + } + return dumpsSorted(item) +} + +// itemKind is bridge_adapter._item_kind: the host's type, "" when it gives none. +func itemKind(entry Obj) string { + raw, _ := get(entry, "item") + item, _ := raw.(Obj) + kind, _ := get(item, "type") + s, _ := kind.(string) + return s +} + +func triples(entries []any) []Item { + out := make([]Item, 0, len(entries)) + for _, raw := range entries { + entry, _ := raw.(Obj) + turn, _ := get(entry, "turnId") + out = append(out, Item{Turn: pyStrOrEmpty(turn), Text: itemText(entry), Type: itemKind(entry)}) + } + return out +} + +// FindToken is find_token: forward paging and honest exhaustion. messageOnly counts only an +// item typed as a user message (or untyped). +func (b BridgeReads) FindToken(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + var cursor any + scanned := 0 + for scanned < limit { + params := Obj{{Key: "threadId", Value: thread}, {Key: "sortDirection", Value: "desc"}, {Key: "limit", Value: int64(min(b.page(), limit-scanned))}} + if cursor != nil { + params = append(params, F{Key: "cursor", Value: cursor}) + } + result, err := b.Call("thread/items/list", params) + if err != nil { + return TokenScan{}, err + } + entries := list(result) + for _, item := range triples(entries) { + scanned++ + if messageOnly && !isMessage(item.Type) { + continue + } + if strings.Contains(item.Text, token) { + var turn any + if item.Turn != "" { + turn = item.Turn + } + return TokenScan{Found: true, TurnID: turn, Scanned: scanned}, nil + } + } + if cursor = cursorOf(result); cursor == nil { + return TokenScan{Exhausted: true, Scanned: scanned}, nil + } + if len(entries) == 0 { + break + } + } + return TokenScan{Scanned: scanned}, nil +} + +// itemPages reads thread/items/list pages lazily, up to limit items, for the token rules. +func (b BridgeReads) itemPages(base Obj, limit int) ([]ItemPage, error) { + var pages []ItemPage + var cursor any + read := 0 + for read < limit { + params := append(append(Obj(nil), base...), F{Key: "limit", Value: int64(min(b.page(), limit-read))}) + if cursor != nil { + params = append(params, F{Key: "cursor", Value: cursor}) + } + result, err := b.Call("thread/items/list", params) + if err != nil { + return nil, err + } + entries := list(result) + read += len(entries) + cursor = cursorOf(result) + pages = append(pages, ItemPage{Items: triples(entries), Follows: cursor != nil}) + if cursor == nil || len(entries) == 0 { + break + } + } + return pages, nil +} + +// FindTokenSince is find_token_since: newest first, stopping at an item of a turn the listing +// showed began before the send. +func (b BridgeReads) FindTokenSince(thread, token string, older []string, limit int) (TokenScan, error) { + pages, err := b.itemPages(Obj{{Key: "threadId", Value: thread}, {Key: "sortDirection", Value: "desc"}}, limit) + if err != nil { + return TokenScan{}, err + } + return FindTokenIn(pages, token, older), nil +} + +// FindTokenInTurn is find_token_in_turn: one turn's own items, oldest first. +func (b BridgeReads) FindTokenInTurn(thread, token, turnID string, limit int) (TokenScan, error) { + pages, err := b.itemPages(Obj{{Key: "threadId", Value: thread}, {Key: "turnId", Value: turnID}, {Key: "sortDirection", Value: "asc"}}, limit) + if err != nil { + return TokenScan{}, err + } + return FindTokenInTurnItems(pages, token, turnID), nil +} diff --git a/internal/relay/delivery/carried_test.go b/internal/relay/delivery/carried_test.go new file mode 100644 index 00000000..67f898eb --- /dev/null +++ b/internal/relay/delivery/carried_test.go @@ -0,0 +1,275 @@ +package delivery + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" +) + +// Carried from todo 25A: test_cli.py CLI-5, CLI-7, CLI-9, CLI-21, CLI-38 (the emit/deliver/ack/ +// verdict half) and test_registration_contention.py RCT-1, RCT-4 (the intent.bind marker-file +// half; the guard-evaluate half is todo 33's). `register` is todo 25's command, so both sides are +// seeded by testdata/cliseed.py through the real Python registry, as TestCLI_every_delivery_... +// does, and every stdout is then compared whole with Python's (stamps and home masked). + +// sameCLI runs one command on both sides and requires the same exit code and the same stdout. +func sameCLI(t *testing.T, py, gosd *cliSide, args ...string) (map[string]any, int) { + t.Helper() + expand := func(s *cliSide) []string { + out := make([]string, len(args)) + for i, a := range args { + out[i] = strings.ReplaceAll(a, "", s.work) + } + return out + } + pout, pcode := py.run(expand(py)...) + gout, gcode := gosd.run(expand(gosd)...) + if pcode != gcode || ageless(py.normal(pout)) != ageless(gosd.normal(gout)) { + t.Fatalf("%v: exit python %d go %d\npython:\n%s\ngo:\n%s", args, pcode, gcode, py.normal(pout), gosd.normal(gout)) + } + var parsed map[string]any + if strings.TrimSpace(pout) != "" { + mustDo(t, json.Unmarshal([]byte(pout), &parsed)) + } + return parsed, pcode +} + +// ageSeconds are status's staged ages, read against the wall clock at the moment each process ran. +var ageSeconds = regexp.MustCompile(`"((?:oldestStaged)?[aA]geSeconds)": [0-9.e-]+`) + +func ageless(text string) string { return ageSeconds.ReplaceAllString(text, `"$1": `) } + +func seededSides(t *testing.T) (*cliSide, *cliSide, string) { + t.Helper() + work := filepath.Join(t.TempDir(), "work") + py, gosd := newSide(t, true, work), newSide(t, false, work) + rid := strings.Trim(sqliteDump(t, py, "SELECT relationship_id FROM relationships"), "[]\"\n ") + if !strings.HasPrefix(rid, "rel-") || !strings.Contains(sqliteDump(t, gosd, "SELECT relationship_id FROM relationships"), rid) { + t.Fatalf("both sides are seeded with the same relationship: %s", rid) + } + return py, gosd, rid +} + +func emitArgs(rid, turn, status, artifact string, extra ...string) []string { + return append([]string{"emit", "--relationship", rid, "--generation", "1", "--outcome", "ready_for_review", "--turn-thread", child, "--turn-id", turn, "--turn-status", status, "--artifact", artifact}, extra...) +} + +func Test25_CLI05_emit_stages_an_unconfirmed_claim_and_status_lists_nothing(t *testing.T) { + t.Run("completed turn offline, then status", func(t *testing.T) { + py, gosd, rid := seededSides(t) + mustDo(t, os.WriteFile(filepath.Join(py.work, "out.txt"), []byte("the deliverable"), 0o644)) + emitted, code := sameCLI(t, py, gosd, emitArgs(rid, dispatchTurn, "completed", "/out.txt")...) + if code != 0 || emitted["stage"] != "staged" || emitted["terminalProof"] != "unverified_staged" || emitted["observedTurnStatus"] != "inProgress" || emitted["receipt"].(map[string]any)["outcome"] != "ready_for_review" { + t.Fatalf("emit %d %v", code, emitted) + } + if _, queued := emitted["delivery"]; queued { + t.Fatal("nothing is queued on an unverified claim") + } + status, _ := sameCLI(t, py, gosd, "status") + if deliveries, _ := status["deliveries"].([]any); len(deliveries) != 0 { + t.Fatalf("a staged claim is not deliverable: %v", status["deliveries"]) + } + }) + t.Run("live inProgress turn", func(t *testing.T) { + py, gosd, rid := seededSides(t) + mustDo(t, os.WriteFile(filepath.Join(py.work, "out.txt"), []byte("still working"), 0o644)) + emitted, _ := sameCLI(t, py, gosd, emitArgs(rid, dispatchTurn, "inProgress", "/out.txt")...) + if _, queued := emitted["delivery"]; queued || emitted["stage"] != "staged" { + t.Fatalf("emit %v", emitted) + } + }) +} + +func Test25_CLI07_a_later_turn_needs_a_continuation(t *testing.T) { + py, gosd, rid := seededSides(t) + mustDo(t, os.WriteFile(filepath.Join(py.work, "out.txt"), []byte("finished later"), 0o644)) + refused, code := sameCLI(t, py, gosd, emitArgs(rid, "turn-loop-5", "completed", "/out.txt")...) + if code != 2 || refused["reason"] != "unassigned_turn" { + t.Fatalf("refused %d %v", code, refused) + } + accepted, code := sameCLI(t, py, gosd, emitArgs(rid, "turn-loop-5", "completed", "/out.txt", + "--continues-anchor", dispatchTurn, "--continuation-actor", "child-loop", "--continuation-reason", "cycle 5 of this execution")...) + if code != 0 || accepted["stage"] != "staged" || accepted["receipt"].(map[string]any)["outcome"] != "ready_for_review" { + t.Fatalf("accepted %d %v", code, accepted) + } +} + +func Test25_CLI09_a_command_needing_the_host_says_so(t *testing.T) { + py, gosd, _ := seededSides(t) + result, code := sameCLI(t, py, gosd, "deliver") + if code != 4 || result["error"] != "usage" || !strings.Contains(result["detail"].(string), "--socket") { + t.Fatalf("deliver %d %v", code, result) + } +} + +func Test25_CLI38_a_malformed_criteria_entry_with_restoration_is_refused(t *testing.T) { + py, gosd, _ := seededSides(t) + for _, criteria := range []string{"[null]", `["c1"]`} { + result, code := sameCLI(t, py, gosd, "verdict", "--event", strings.Repeat("e", 32), "--verdict", "needs_changes", "--verdict-turn", "v1", "--criteria", criteria, "--restoration", "c1") + if code != 2 || result["reason"] != "disposition_conflict" { + t.Fatalf("%s: %d %v", criteria, code, result) + } + } +} + +// CLI-21: the CLI's verdict carries its sync outbox obligation. The Go `verdict` command wires +// VerdictSync before it rules (there is no lazily built service to forget it on); this proves +// the wiring through the real command: the same sync_outbox row as Python's for one verdict. +func Test25_CLI21_the_verdict_command_is_wired_to_the_outbox(t *testing.T) { + py, gosd, rid := seededSides(t) + event := "" + for _, side := range []*cliSide{py, gosd} { + // The same acknowledged completion on both sides, through the real Python services, and + // the coordination-document target the outbox answers to. + script := ` +import sys +from pathlib import Path +from codex_session_relay import identity +from codex_session_relay.clock import FakeClock +from codex_session_relay.ack import AckService +from codex_session_relay.delivery import DeliveryService +from codex_session_relay.fakehost import FakeHostAdapter +from codex_session_relay.receipts import ReceiptIntake, TurnRef +from codex_session_relay.registry import Registry, record_settings +from tests.support import task_settings +from codex_session_relay import manifest +from codex_session_relay.store import Store +state, work, rid = sys.argv[1:4] +store = Store(state + "/relay.sqlite3"); clock = FakeClock(); registry = Registry(store, clock) +intake = ReceiptIntake(store, registry, clock) +delivery = DeliveryService(store, registry, intake, clock) +ack = AckService(store, registry, intake, delivery, clock) +record_settings(store, clock, "01parent-task", task_settings("/parent"), source="creation_result") +record_settings(store, clock, "01child-task", task_settings(work), source="creation_result") +adapter = FakeHostAdapter(clock); adapter.add_thread("01parent-task"); adapter.add_thread("01child-task") +Path(work, "out.txt").write_text("the deliverable") +entries, _ = manifest.build([work + "/out.txt"], [work]) +digest = manifest.revision_hash(entries) +event = identity.event_id(rid, 1, digest, "ready_for_review", turn_id="turn-dispatch-1", attempt=1) +payload = {"eventId": event, "relationshipId": rid, "executionGeneration": 1, "attempt": 1, "revisionHash": digest, + "outcome": "ready_for_review", "producer": "child", "turnRef": {"threadId": "01child-task", "turnId": "turn-dispatch-1", "turnStatus": "completed"}, + "manifest": [{"path": e.path, "sha256": e.sha256, "bytes": e.bytes} for e in entries], "emittedAt": clock.iso()} +intake.accept_child_receipt(payload, observation=TurnRef("01child-task", "turn-dispatch-1", "completed")) +delivery.enqueue(event); delivery.attempt(event, adapter); clock.advance(5) +turn = adapter.start_turn("01parent-task", turn_id="ack-turn", status="inProgress") +ack.acknowledge(event, ack_turn_id="ack-turn", ack_proof=identity.ack_proof(event, "ack-turn"), accepted=True, adapter=adapter) +store.db.execute("INSERT INTO sync_targets (relationship_id, target, target_ref, recorded_at) VALUES (?,?,?,?)", (rid, "coordination_document", "DOC-1", clock.iso())) +store.db.commit() if store.db.in_transaction else None +store.close() +print(event) +` + out, err := execUV(side, "python", "-c", script, side.state, side.work, rid).CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + event = strings.TrimSpace(string(out)) + } + record, code := sameCLI(t, py, gosd, "verdict", "--event", event, "--verdict", "verified", "--verdict-turn", "v1") + if code != 0 || record["verdict"] != "verified" { + t.Fatalf("verdict %d %v", code, record) + } + query := "SELECT sync_id, relationship_id, target, target_ref, subject_kind, event_id, verdict FROM sync_outbox ORDER BY 1" + pr, gr := sqliteDump(t, py, query), sqliteDump(t, gosd, query) + if pr != gr || !strings.Contains(pr, event) { + t.Fatalf("sync_outbox differs or is empty\npython: %s\ngo: %s", pr, gr) + } +} + +// ---------------------------------------------------------------- test_registration_contention.py + +// RCT-1 (marker half): a claim published before the bind, then the late bind and the relationship +// registration, leave the marker facts the fold reads: one claim, one bound identity, the +// registration, and the derived state moving from claimed-unbound to registered. The Stop +// decisions and hold files around it are guard.evaluate (todo 33). +func Test25_RCT01_a_late_bind_lands_on_the_claimed_assignment(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "claim", "session": session1}, + markerOp{"op": "state"}, + markerOp{"op": "correlated", "session": session1}, + bindOp(), openOp(), registerOp(), + markerOp{"op": "state"}, + markerOp{"op": "facts"}, + ) + if ok(t, answers[2]) == ok(t, answers[7]) { + t.Fatalf("the late bind and registration did not move the assignment: %v -> %v", answers[2], answers[7]) + } + facts := ok(t, answers[8]).(map[string]any)["facts"].(map[string]any) + if facts["bound"].(map[string]any)["sessionId"] != session1 || len(facts["claims"].([]any)) != 1 { + t.Fatalf("facts %v", facts) + } +} + +// RCT-4: two concurrent binds on one assignment, from two goroutines released together: one +// bound and one conflict, one recorded conflict naming the loser, the loser told the winner, +// state identity_bound and contested until a resolution naming the winner. +func Test25_RCT04_two_concurrent_binds_leave_one_winner_and_one_recorded_conflict(t *testing.T) { + // The sequential shape of the same outcome, compared whole with Python. + sameOps(t, nil, + declareOp(), bindOp(), markerOp{"op": "bind", "session": "01other-session", "task": "01other-task"}, + markerOp{"op": "state"}, markerOp{"op": "contested"}, + markerOp{"op": "resolution", "facts": []any{"conflicts/0"}, "task": task1, "session": session1}, + markerOp{"op": "contested"}, + ) + // The race itself, in Go. + root, work := filepath.Join(t.TempDir(), "markers"), t.TempDir() + t.Setenv(MarkerEnv, "") + _, err := DeclareIntent(root, IntentDeclaration{Workspace: work, DispatchRequestID: "dispatch-request-1", IssueKey: "REL-1", DeclaredAt: "2026-01-01T00:00:00+00:00"}) + mustDo(t, err) + assignment := AssignmentID("dispatch-request-1") + const at = "2026-01-01T00:00:00+00:00" + outcomes := make([]Obj, 2) + errs := make([]error, 2) + var ready, done sync.WaitGroup + start := make(chan struct{}) + for i, who := range [][2]string{{child, child}, {"01other-session", "01other-task"}} { + ready.Add(1) + done.Add(1) + go func() { + defer done.Done() + ready.Done() + <-start + outcomes[i], errs[i] = BindIdentity(root, work, assignment, who[0], who[1], at) + }() + } + ready.Wait() + close(start) + done.Wait() + for _, err := range errs { + mustDo(t, err) + } + results := []string{str(outcomes[0], "outcome"), str(outcomes[1], "outcome")} + if !(results[0] == Bound && results[1] == Conflict) && !(results[0] == Conflict && results[1] == Bound) { + t.Fatalf("two concurrent binds must settle as one winner and one conflict: %v", outcomes) + } + directory, err := AssignmentDir(root, work, assignment) + mustDo(t, err) + facts, _ := ReadAssignment(directory) + bound := sub(facts, "bound") + conflicts, _ := field(facts, "conflicts").([]any) + if len(conflicts) != 1 || str(conflicts[0].(Obj), "attemptedSessionId") == str(bound, "sessionId") { + t.Fatalf("one recorded conflict naming the loser: %v", conflicts) + } + loser := outcomes[0] + if results[1] == Conflict { + loser = outcomes[1] + } + if str(loser, "boundSessionId") != str(bound, "sessionId") { + t.Fatalf("the loser is told the winner: %v", loser) + } + if DeriveAssignmentState(facts, at) != IdentityBound || !IdentityContested(facts) { + t.Fatalf("state %s contested %v", DeriveAssignmentState(facts, at), IdentityContested(facts)) + } + fact := conflicts[0].(Obj) + _, err = PublishResolution(root, work, assignment, str(bound, "taskId"), str(bound, "sessionId"), "the winning link() is the identity", at, + []Obj{{{Key: "factId", Value: field(fact, "factId")}, {Key: "digest", Value: FactDigest(fact)}}}) + mustDo(t, err) + resolved, _ := ReadAssignment(directory) + if IdentityContested(resolved) { + t.Fatal("a resolution naming the bound identity did not settle the contest") + } +} diff --git a/internal/relay/delivery/cli.go b/internal/relay/delivery/cli.go new file mode 100644 index 00000000..635cd33c --- /dev/null +++ b/internal/relay/delivery/cli.go @@ -0,0 +1,567 @@ +package delivery + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/contract" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The twelve delivery commands of `codex-session-relay` (cli.py: emit, deliver, reconcile, +// recover, claim, ack-proof, ack, verdict, criteria-register, criteria-show, revision-head, +// verify-acks), with argparse's parsing rules and main()'s JSON replies and exit codes. + +type flagSpec struct { + name string + kind string // store, int, append, true + required bool + choices []string + def any +} + +type commandSpec struct { + flags []flagSpec + run func(*cliRun) (any, error) + // exempt is _reads_no_selected_store: the command answers without the store discovery picks, + // so the selection refusal is not asked. + exempt func(map[string]any) bool +} + +var deliveryCommands = map[string]commandSpec{ + "emit": {flags: []flagSpec{{"--relationship", "store", true, nil, nil}, {"--generation", "int", true, nil, nil}, {"--attempt", "int", false, nil, int64(1)}, + {"--outcome", "store", true, []string{"ready_for_review", "failed", "interrupted", "blocked_needs_input"}, nil}, {"--turn-thread", "store", true, nil, nil}, {"--turn-id", "store", true, nil, nil}, + {"--turn-status", "store", false, []string{"completed", "failed", "interrupted", "inProgress"}, "inProgress"}, {"--artifact", "append", false, nil, nil}, {"--manifest-ref", "store", false, nil, nil}, + {"--continues-anchor", "store", false, nil, nil}, {"--continuation-actor", "store", false, nil, nil}, {"--continuation-reason", "store", false, nil, nil}, {"--supersedes-revision", "store", false, nil, nil}, {"--no-enqueue", "true", false, nil, false}}, + run: cmdEmit}, + "deliver": {flags: []flagSpec{{"--event", "store", false, nil, nil}, {"--limit", "int", false, nil, int64(4)}}, run: needsHost}, + "reconcile": {flags: []flagSpec{{"--request-id", "store", true, nil, nil}}, run: needsHost}, + "recover": {run: needsHost}, + "claim": {flags: []flagSpec{{"--event", "store", true, nil, nil}, {"--turn", "store", false, nil, nil}}, run: cmdClaim}, + "ack-proof": {flags: []flagSpec{{"--event", "store", true, nil, nil}, {"--turn", "store", true, nil, nil}}, run: cmdAckProof}, + "ack": {flags: []flagSpec{{"--event", "store", true, nil, nil}, {"--ack-turn", "store", true, nil, nil}, {"--ack-proof", "store", true, nil, nil}, {"--reject", "store", false, nil, nil}}, run: cmdAck}, + "verify-acks": {flags: []flagSpec{{"--limit", "int", false, nil, int64(8)}}, run: needsHost}, + "verdict": {flags: []flagSpec{{"--event", "store", true, nil, nil}, {"--verdict", "store", true, []string{"verified", "needs_changes", "unverified", "aborted"}, nil}, {"--verdict-turn", "store", true, nil, nil}, + {"--criterion", "append", false, nil, nil}, {"--finding", "append", false, nil, nil}, {"--criteria", "store", false, nil, nil}, {"--restoration", "store", false, nil, nil}, {"--reason", "store", false, nil, nil}, {"--expect-criteria-digest", "store", false, nil, nil}}, + run: cmdVerdict}, + "criteria-register": {flags: []flagSpec{{"--relationship", "store", true, nil, nil}, {"--criterion", "append", true, nil, nil}, {"--optional", "append", false, nil, nil}, {"--source-ref", "store", false, nil, nil}}, run: cmdCriteriaRegister}, + "criteria-show": {flags: []flagSpec{{"--relationship", "store", true, nil, nil}}, run: cmdCriteriaShow}, + "revision-head": {flags: []flagSpec{{"--relationship", "store", true, nil, nil}, {"--generation", "int", false, nil, nil}}, run: cmdRevisionHead}, +} + +// usageError is SystemExit2: a JSON usage reply with its exit code. +type usageError struct { + detail string + code int +} + +func (u *usageError) Error() string { return u.detail } + +type cliRun struct { + ctx context.Context + args map[string]any + state string + socket string + store *store.Store + clock Clock +} + +func (c *cliRun) s(name string) string { v, _ := c.args[name].(string); return v } +func (c *cliRun) opt(name string) any { return c.args[name] } +func (c *cliRun) list(name string) []string { v, _ := c.args[name].([]string); return v } + +func (c *cliRun) openStore() (*store.Store, error) { + if c.store == nil { + s, err := store.Open(c.ctx, c.state+"/relay.sqlite3", c.socket) + if err != nil { + return nil, err + } + c.store = s + } + return c.store, nil +} + +func (c *cliRun) services() (*Service, *Ack, error) { + s, err := c.openStore() + if err != nil { + return nil, nil, err + } + d := NewService(s, c.clock) + return d, NewAck(d), nil +} + +// ParseDeliveryArgs is argparse for one command: the parsed values, or its error message. +func parseArgs(spec commandSpec, argv []string) (map[string]any, string) { + out := map[string]any{} + for _, f := range spec.flags { + out[f.name] = f.def + } + seen := map[string]bool{} + var unknown []string + for i := 0; i < len(argv); i++ { + arg := argv[i] + name, value, hasValue := strings.Cut(arg, "=") + var f *flagSpec + for j := range spec.flags { + if spec.flags[j].name == name { + f = &spec.flags[j] + } + } + if f == nil { + unknown = append(unknown, arg) + continue + } + seen[f.name] = true + if f.kind == "true" { + out[f.name] = true + continue + } + if !hasValue { + if i+1 >= len(argv) || strings.HasPrefix(argv[i+1], "--") { + return nil, fmt.Sprintf("argument %s: expected one argument", f.name) + } + i++ + value = argv[i] + } + if f.choices != nil && !slices.Contains(f.choices, value) { + quoted := make([]string, len(f.choices)) + for k, c := range f.choices { + quoted[k] = "'" + c + "'" + } + return nil, fmt.Sprintf("argument %s: invalid choice: '%s' (choose from %s)", f.name, value, strings.Join(quoted, ", ")) + } + switch f.kind { + case "int": + n, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return nil, fmt.Sprintf("argument %s: invalid int value: '%s'", f.name, value) + } + out[f.name] = n + case "append": + list, _ := out[f.name].([]string) + out[f.name] = append(list, value) + default: + out[f.name] = value + } + } + var missing []string + for _, f := range spec.flags { + if f.required && !seen[f.name] { + missing = append(missing, f.name) + } + } + if len(missing) > 0 { + return nil, "the following arguments are required: " + strings.Join(missing, ", ") + } + if len(unknown) > 0 { + return nil, "unrecognized arguments: " + strings.Join(unknown, " ") + } + return out, "" +} + +// SelectionCheck is cli._refuse_ambiguous_state for the resolved selection: nil, or an error +// carrying the refusal payload (PayloadError). The relay CLI passes its own, so the recovery +// lines name the program as it was invoked. +type SelectionCheck func(selection store.StateSelection, socket string) error + +// PayloadError is an answer printed whole with its own exit code (cli.PayloadExit). +type PayloadError interface { + error + ExitPayload() (contract.OrderedObject, int) +} + +// ExecuteCLI runs one delivery command as the codex-session-relay console script, with this +// package's own selection refusal. handled is false for any other command. +func ExecuteCLI(ctx context.Context, argv []string, stdout, stderr io.Writer) (int, bool) { + return ExecuteAs(ctx, "codex-session-relay", argv, stdout, stderr, nil) +} + +// ExecuteAs runs one delivery command of the relay CLI, as cli.main does for it: prog is the +// program name argparse prints, check the selection refusal applied before the handler (nil +// uses this package's). handled is false for any other command. +func ExecuteAs(ctx context.Context, prog string, argv []string, stdout, stderr io.Writer, check SelectionCheck) (int, bool) { + var state, socket string + i := 0 + for ; i < len(argv); i++ { + name, value, has := strings.Cut(argv[i], "=") + switch name { + case "--json": + continue + case "--state", "--socket", "--kind-module": + if !has { + if i+1 >= len(argv) { + return 0, false + } + i++ + value = argv[i] + } + if name == "--state" { + state = value + } else if name == "--socket" { + socket = value + } + continue + } + break + } + if i >= len(argv) { + return 0, false + } + command := argv[i] + spec, ok := deliveryCommands[command] + if !ok { + return 0, false + } + parsed, problem := parseArgs(spec, argv[i+1:]) + if problem != "" { + fmt.Fprintf(stderr, "usage: %s %s [-h] ...\n%s %s: error: %s\n", prog, command, prog, command, problem) + return 2, true + } + run := &cliRun{ctx: ctx, args: parsed, socket: socket, clock: SystemClock{}} + if command != "ack-proof" { + selection, err := store.ResolveStateDir(state, socket) + if err != nil { + return reply(stdout, Obj{{Key: "error", Value: "host"}, {Key: "detail", Value: "OSError: " + err.Error()}}, contract.ExitHost), true + } + if spec.exempt != nil && spec.exempt(parsed) { + // Marker-only: no selection refusal (cli._refuse_ambiguous_state's exemption). + } else if check != nil { + if err := check(selection, socket); err != nil { + var payload PayloadError + if errors.As(err, &payload) { + body, code := payload.ExitPayload() + return reply(stdout, body, code), true + } + return reply(stdout, Obj{{Key: "error", Value: "host"}, {Key: "detail", Value: hostDetail(err)}}, contract.ExitHost), true + } + } else if refusal := selectionRefusal(selection, socket); refusal != nil { + return reply(stdout, refusal, contract.ExitRefused), true + } + run.state = selection.Path + } + defer func() { + if run.store != nil { + _ = run.store.Close() + } + }() + result, err := spec.run(run) + var usage *usageError + var refused *store.RefusedError + var whole PayloadError + switch { + case errors.As(err, &usage): + return reply(stdout, Obj{{Key: "error", Value: "usage"}, {Key: "detail", Value: usage.detail}}, usage.code), true + case errors.As(err, &whole): + body, code := whole.ExitPayload() + return reply(stdout, body, code), true + case errors.As(err, &refused): + return reply(stdout, Obj{{Key: "error", Value: "refused"}, {Key: "reason", Value: refused.Reason}, {Key: "detail", Value: refused.Detail}}, contract.ExitRefused), true + case err != nil: + return reply(stdout, Obj{{Key: "error", Value: "host"}, {Key: "detail", Value: hostDetail(err)}}, contract.ExitHost), true + } + return reply(stdout, result, contract.ExitOk), true +} + +// hostError carries the Python exception class name of a host failure. +type hostError struct{ kind, message string } + +func (h *hostError) Error() string { return h.kind + ": " + h.message } + +func hostDetail(err error) string { + var h *hostError + if errors.As(err, &h) { + return h.Error() + } + return "RuntimeError: " + err.Error() +} + +func reply(w io.Writer, value any, code int) int { + if err := contract.Emit(w, value); err != nil { + fmt.Fprintln(os.Stderr, err) + return contract.ExitHost + } + return code +} + +// needsHost is the four commands that reach the App Server. Without --socket they are a usage +// error, as in Python; with it, the host adapter they drive is the bridge adapter port (todo 28). +func needsHost(c *cliRun) (any, error) { + if c.socket == "" { + return nil, &usageError{"this command needs --socket to reach the host", contract.ExitUsage} + } + return nil, &hostError{"HostUnavailable", "the relay host adapter (bridge_adapter.py) is not ported to Go yet (todo 28)"} +} + +var eventIDPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) + +func cmdAckProof(c *cliRun) (any, error) { + event, turn := c.s("--event"), c.s("--turn") + if !eventIDPattern.MatchString(event) { + return nil, &hostError{"ValueError", "event id must be 32 lowercase hex characters"} + } + if strings.TrimSpace(turn) == "" { + return nil, &hostError{"ValueError", "ack_turn_id must be a non-empty string"} + } + return Obj{{Key: "eventId", Value: event}, {Key: "turnId", Value: turn}, {Key: "ackProof", Value: AckProof(event, turn)}}, nil +} + +func cmdClaim(c *cliRun) (any, error) { + _, ack, err := c.services() + if err != nil { + return nil, err + } + claim, err := ack.ClaimVerification(c.ctx, c.s("--event"), c.opt("--turn")) + return Obj{{Key: "claim", Value: claim}}, err +} + +func cmdAck(c *cliRun) (any, error) { + _, ack, err := c.services() + if err != nil { + return nil, err + } + return AckCommand(c.ctx, ack, nil, nil, c.s("--event"), c.s("--ack-turn"), c.s("--ack-proof"), c.opt("--reject")) +} + +// AckCommand is cmd_ack. With a host adapter and reconciler, a delivery the relay has not +// confirmed is reconciled first through the acknowledging turn (the proof is checked first, +// locally, so a wrong one makes no host read). +func AckCommand(ctx context.Context, ack *Ack, rc *Reconciler, adapter Adapter, event, ackTurn, proof string, reject any) (Obj, error) { + if adapter != nil && rc != nil && proof == AckProof(event, ackTurn) { + if _, err := rc.ConfirmDelivery(ctx, event, adapter, ackTurn); err != nil { + return nil, err + } + } + record, err := ack.Acknowledge(ctx, event, ackTurn, proof, !truthy(reject), reject, adapter) + if err != nil { + return nil, err + } + if why, ok := get(record, "_deliveryUnconfirmed"); ok && truthy(why) { + record = append(record, F{Key: "_note", Value: "kept as the parent's authored acknowledgement: the relay could not yet confirm this delivery for the turn it read (" + pyStr(why) + "). The daemon completes it once the delivery is confirmed, and a verdict completes it first; nothing needs to be acknowledged or sent again."}) + } else if str(record, "_verified") != "verified" { + record = append(record, F{Key: "_note", Value: "recorded as the parent's authored intent; this turn is not established yet, so it does not close the attempt and cannot yet produce a verdict. Run verify-acks from a process with host access."}) + } + return record, nil +} + +// CompleteKeptAcknowledgement is _complete_kept_acknowledgement: before a verdict, a kept +// acknowledgement's delivery is confirmed through its turn and the acknowledgement completed. +func CompleteKeptAcknowledgement(ctx context.Context, ack *Ack, rc *Reconciler, adapter Adapter, event string) error { + if adapter == nil { + return nil + } + turn, err := ack.KeptTurn(ctx, event) + if err != nil || turn == "" { + return err + } + if rc != nil { + if _, err := rc.ConfirmDelivery(ctx, event, adapter, turn); err != nil { + return err + } + } + _, err = ack.CompletePending(ctx, event, adapter) + return err +} + +// VerifyAcksCommand is cmd_verify_acks: kept acknowledgements have their delivery confirmed +// through their turn first, then the pending pass completes what it can. +func VerifyAcksCommand(ctx context.Context, ack *Ack, rc *Reconciler, adapter Adapter, limit int) (Obj, error) { + confirmations := []any{} + kept, err := ack.KeptUnconfirmed(ctx, ack.Clock.Now(), limit) + if err != nil { + return nil, err + } + for _, k := range kept { + outcome, err := rc.ConfirmDelivery(ctx, k[0], adapter, k[1]) + if err != nil { + return nil, err + } + if outcome == nil { + continue + } + picked := Obj{} + for _, key := range []string{"eventId", "requestId", "confirmed", "turnRead", "error"} { + if v, ok := get(outcome, key); ok { + picked = append(picked, F{Key: key, Value: v}) + } + } + confirmations = append(confirmations, picked) + } + results, err := ack.VerifyPendingAcks(ctx, adapter, limit, nil) + if err != nil { + return nil, err + } + out := Obj{{Key: "results", Value: results}} + if len(confirmations) > 0 { + out = append(out, F{Key: "confirmations", Value: confirmations}) + } + return out, nil +} + +func cmdCriteriaRegister(c *cliRun) (any, error) { + d, ack, err := c.services() + if err != nil { + return nil, err + } + _ = d + optional := c.list("--optional") + var entries []any + for _, item := range c.list("--criterion") { + id, title, _ := strings.Cut(item, "=") + entries = append(entries, Obj{{Key: "id", Value: strings.TrimSpace(id)}, {Key: "title", Value: strings.TrimSpace(title)}, {Key: "required", Value: !slices.Contains(optional, strings.TrimSpace(id))}}) + } + return ack.Criteria.Register(c.ctx, c.s("--relationship"), entries, c.opt("--source-ref")) +} + +func cmdCriteriaShow(c *cliRun) (any, error) { + _, ack, err := c.services() + if err != nil { + return nil, err + } + rid := c.s("--relationship") + registered, err := ack.Criteria.Get(c.ctx, rid) + if err != nil { + return nil, err + } + mode, err := ack.Criteria.Mode(c.ctx, rid) + if err != nil { + return nil, err + } + var criteria, digest, source any + if registered != nil { + criteria, _ = get(registered, "criteria") + digest, _ = get(registered, "setDigest") + source, _ = get(registered, "sourceRef") + } + return Obj{{Key: "relationshipId", Value: rid}, {Key: "mode", Value: mode}, {Key: "criteria", Value: criteria}, {Key: "setDigest", Value: digest}, {Key: "sourceRef", Value: source}}, nil +} + +func cmdRevisionHead(c *cliRun) (any, error) { + d, _, err := c.services() + if err != nil { + return nil, err + } + rid := c.s("--relationship") + r, err := LoadRelationship(c.ctx, d.Store, rid) + if err != nil { + return nil, err + } + generation := r.Generation + if g, ok := c.opt("--generation").(int64); ok && g != 0 { + generation = g + } + head, err := HeadRevision(c.ctx, d.Store, rid, generation) + return Obj{{Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: generation}, {Key: "head", Value: head}}, err +} + +func cmdVerdict(c *cliRun) (any, error) { + var criteria, findings []any + for _, item := range c.list("--criterion") { + name, value, _ := strings.Cut(item, "=") + if value == "" { + value = "verified" + } + criteria = append(criteria, Obj{{Key: "id", Value: name}, {Key: "verdict", Value: value}}) + } + for _, item := range c.list("--finding") { + name, rest, _ := strings.Cut(item, "=") + disposition, note, _ := strings.Cut(rest, ":") + if disposition == "" { + disposition = "verified" + } + findings = append(findings, Obj{{Key: "id", Value: name}, {Key: "verdict", Value: disposition}, {Key: "note", Value: strings.TrimSpace(note)}}) + } + if raw := c.s("--criteria"); c.opt("--criteria") != nil { + text := raw + if strings.HasPrefix(raw, "@") { + content, err := os.ReadFile(raw[1:]) + if err != nil { + return nil, &hostError{"FileNotFoundError", err.Error()} + } + text = string(content) + } + parsed, err := loads(text) + if err != nil { + return nil, &hostError{"JSONDecodeError", err.Error()} + } + switch v := parsed.(type) { + case []any: + findings = append(findings, v...) + case Obj: + for _, f := range v { + findings = append(findings, f.Key) + } + default: + return nil, &hostError{"TypeError", fmt.Sprintf("'%s' object is not iterable", pyTypeName(v))} + } + } + if c.opt("--restoration") != nil { + wanted := strings.TrimSpace(c.s("--restoration")) + if wanted == "" { + return nil, &usageError{"--restoration names the criterion id whose finding carries the block, so it cannot be empty. Leave the option out to carry no block", contract.ExitUsage} + } + all := append(append([]any(nil), criteria...), findings...) + var marked []Obj + wellFormed := true + for _, item := range all { + o, ok := item.(Obj) + if !ok { + wellFormed = false + continue + } + if id, _ := get(o, "id"); strings.TrimSpace(pyStrOrEmpty(id)) == wanted { + marked = append(marked, o) + } + } + if len(marked) == 0 && wellFormed { + return nil, &usageError{"--restoration names " + store.PyRepr(c.s("--restoration")) + ", which is not one of the findings this verdict carries. The block travels inside a finding, so it names one", contract.ExitUsage} + } + for _, list := range [][]any{criteria, findings} { + for i, item := range list { + o, ok := item.(Obj) + if !ok { + continue + } + if id, _ := get(o, "id"); strings.TrimSpace(pyStrOrEmpty(id)) != wanted { + continue + } + existing, present := get(o, "restoration") + if present && existing == false { + return nil, &usageError{"--restoration names " + store.PyRepr(wanted) + ", whose finding declares the restoration block false. One correction carries one block and says so once", contract.ExitUsage} + } + if present && existing != nil { + if _, isBool := existing.(bool); !isBool { + continue + } + } + list[i] = set(o, "restoration", true) + } + } + } + _, ack, err := c.services() + if err != nil { + return nil, err + } + ack.Sync = VerdictSync(ack.Store, ack.Clock) + event := c.s("--event") + record, err := ack.RecordVerdict(c.ctx, event, c.s("--verdict"), c.s("--verdict-turn"), criteria, findings, c.opt("--reason"), c.opt("--expect-criteria-digest")) + if err != nil { + return nil, err + } + restoration, err := ack.RestorationOf(c.ctx, event) + if err != nil { + return nil, err + } + return append(record, F{Key: "_restoration", Value: restoration}), nil +} + +// CommandNames are the relay subcommands this package serves, in cli.py's add_parser order. +func CommandNames() []string { + return append([]string{"emit", "deliver", "reconcile", "recover", "claim", "ack-proof", "ack", "verdict", "criteria-register", "criteria-show", "revision-head", "verify-acks"}, intentCommandNames...) +} diff --git a/internal/relay/delivery/cli_emit.go b/internal/relay/delivery/cli_emit.go new file mode 100644 index 00000000..251d14d3 --- /dev/null +++ b/internal/relay/delivery/cli_emit.go @@ -0,0 +1,122 @@ +package delivery + +import ( + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// cmdEmit is cmd_emit: the child's receipt, accepted, and queued when final. +func cmdEmit(c *cliRun) (any, error) { + d, _, err := c.services() + if err != nil { + return nil, err + } + rid := c.s("--relationship") + relationship, err := RequireActive(c.ctx, d.Store, rid) + if err != nil { + return nil, err + } + generation := c.opt("--generation").(int64) + attempt := int(c.opt("--attempt").(int64)) + outcome := c.s("--outcome") + var manifest any + digest := store.NoDeliverable + var entries []store.ManifestEntry + if paths := c.list("--artifact"); len(paths) > 0 { + if entries, err = store.BuildManifest(paths, relationship.ArtifactRoots); err != nil { + return nil, err + } + if digest, err = store.ManifestRevision(entries); err != nil { + return nil, err + } + records := make([]any, len(entries)) + for i, e := range entries { + o := Obj{{Key: "path", Value: e.Path}, {Key: "sha256", Value: e.SHA256}} + if e.Bytes != nil { + o = append(o, F{Key: "bytes", Value: *e.Bytes}) + } + records[i] = o + } + manifest = records + } + reference := c.s("--manifest-ref") + if reference != "" && len(entries) > 0 { + if err := store.FreezeManifest(entries, reference); err != nil { + return nil, err + } + } + // No host in this process (the adapter is todo 28's): a readiness claim may only stage. + status, proof := c.s("--turn-status"), "claimed" + if c.socket != "" { + return nil, &hostError{"HostUnavailable", "the relay host adapter (bridge_adapter.py) is not ported to Go yet (todo 28)"} + } + if outcome == "ready_for_review" && status != "inProgress" { + status, proof = "inProgress", "unverified_staged" + } + event, err := store.EventID(rid, int(generation), digest, outcome, c.s("--turn-id"), &attempt) + if err != nil { + return nil, &hostError{"ValueError", "event identity: " + err.Error()} + } + thread, turn := c.s("--turn-thread"), c.s("--turn-id") + payload := Obj{{Key: "eventId", Value: event}, {Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: generation}, {Key: "attempt", Value: int64(attempt)}, + {Key: "revisionHash", Value: digest}, {Key: "outcome", Value: outcome}, {Key: "producer", Value: "child"}, + {Key: "turnRef", Value: Obj{{Key: "threadId", Value: thread}, {Key: "turnId", Value: turn}, {Key: "turnStatus", Value: status}}}, + {Key: "manifest", Value: manifest}, {Key: "emittedAt", Value: c.clock.ISO()}} + if reference != "" { + payload = append(payload, F{Key: "manifestRef", Value: reference}) + } + options := store.AcceptOptions{} + if anchor := c.s("--continues-anchor"); anchor != "" { + actor := c.s("--continuation-actor") + if actor == "" { + actor = thread + } + reason := c.s("--continuation-reason") + if reason == "" { + reason = "continuation of this execution" + } + options.Continuation = []byte(dumps(Obj{{Key: "anchorTurnId", Value: anchor}, {Key: "actor", Value: actor}, {Key: "reason", Value: reason}})) + } + if c.opt("--supersedes-revision") != nil { + s := c.s("--supersedes-revision") + options.SupersedesRevision = &s + } + intake := store.ReceiptIntake{Store: d.Store, Now: c.clock.ISO, Minimum: store.BestEffortDetection} + stored, err := intake.AcceptChildReceiptWith(c.ctx, []byte(dumps(payload)), store.TurnReference{ThreadID: thread, TurnID: turn, Status: status}, options) + if err != nil { + return nil, err + } + receipt := Obj{} + for _, f := range loadsObj(stored.Record) { + if len(f.Key) > 0 && f.Key[0] == '_' || (f.Key == "manifestRef" || f.Key == "criteria") && f.Value == nil { + continue + } + receipt = append(receipt, f) + } + result := Obj{{Key: "receipt", Value: receipt}, {Key: "stage", Value: stored.Stage}, {Key: "duplicate", Value: stored.Duplicate}, {Key: "terminalProof", Value: proof}, {Key: "observedTurnStatus", Value: status}} + if stored.Stage == "final" { + if err := d.AnnotatePredecessors(c.ctx, event); err != nil { + return nil, err + } + existing, err := d.Find(c.ctx, event) + if err != nil { + return nil, err + } + if c.opt("--no-enqueue") != true && existing == nil { + row, err := d.Enqueue(c.ctx, event, "", "") + if err != nil { + return nil, err + } + result = append(result, F{Key: "delivery", Value: rowObj(row)}) + } + } + return result, nil +} + +// rowObj is dict(sqlite3.Row) of a deliveries row, in the table's column order. +func rowObj(row Row) Obj { + var out Obj + for _, column := range []string{"event_id", "relationship_id", "kind", "recipient_task_id", "recipient_thread_id", "state", "attempt_count", "next_eligible_at", "hold_reason", "lease_owner", "lease_until", "dispatch_evidence", "dispatch_turn_id", "provenance", "created_at", "updated_at"} { + out = append(out, F{Key: column, Value: row.Opt(column)}) + } + return out +} diff --git a/internal/relay/delivery/cli_test.go b/internal/relay/delivery/cli_test.go new file mode 100644 index 00000000..d50ab578 --- /dev/null +++ b/internal/relay/delivery/cli_test.go @@ -0,0 +1,197 @@ +package delivery + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The twelve delivery commands through the real processes: Python's `codex-session-relay` and +// the built `crw relay`, each over its own copy of the same seeded store. Every stdout is +// compared whole, byte for byte, after the wall-clock stamps (the only nondeterministic bytes) +// are replaced by one token; exit codes are compared exactly. + +var ( + crwOnce sync.Once + crwPath string + crwErr error +) + +func crwBinary(t *testing.T) string { + t.Helper() + crwOnce.Do(func() { + dir, err := os.MkdirTemp("", "crw-delivery-cli-") + if err != nil { + crwErr = err + return + } + crwPath = filepath.Join(dir, "crw") + build := exec.Command("go", "build", "-buildvcs=false", "-o", crwPath, "./cmd/crw") + build.Dir = repoRoot(t) + if out, err := build.CombinedOutput(); err != nil { + crwErr = err + t.Log(string(out)) + } + }) + if crwErr != nil { + t.Fatal(crwErr) + } + return crwPath +} + +type cliSide struct { + t *testing.T + home string + state string + work string + argv0 []string + dir string + env []string +} + +// newSide keeps the artifact tree at one shared path (work), because a revision hash covers the +// declared path; each side has its own home and store. +func newSide(t *testing.T, python bool, work string) *cliSide { + home := t.TempDir() + s := &cliSide{t: t, home: home, state: filepath.Join(home, "state"), work: work} + root := repoRoot(t) + s.env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "xs"), "XDG_DATA_HOME="+filepath.Join(home, "xd"), "XDG_CONFIG_HOME="+filepath.Join(home, "xc"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, "CODEX_SESSION_RELAY_STATE=") + if python { + s.argv0 = []string{"uv", "run", "--no-sync", "codex-session-relay"} + s.dir = filepath.Join(root, "packages", "codex-session-relay") + } else { + s.argv0 = []string{crwBinary(t), "relay"} + s.dir = root + } + seed := exec.Command("uv", "run", "--no-sync", "python", filepath.Join(root, "internal/relay/delivery/testdata/cliseed.py"), s.state, s.work) + seed.Dir = filepath.Join(root, "packages", "codex-session-relay") + seed.Env = s.env + out, err := seed.Output() + mustDo(t, err) + if strings.TrimSpace(string(out)) != "rel-4675b3fb54d7b85d" && !strings.HasPrefix(string(out), "rel-") { + t.Fatalf("seed %s", out) + } + return s +} + +func (s *cliSide) run(args ...string) (string, int) { + argv := append(append([]string(nil), s.argv0...), append([]string{"--state", s.state}, args...)...) + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Dir = s.dir + cmd.Env = s.env + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + code := 0 + if e, ok := err.(*exec.ExitError); ok { + code = e.ExitCode() + } else if err != nil { + s.t.Fatal(err) + } + return out.String(), code +} + +var stamp = regexp.MustCompile(`\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}\+00:00`) + +func (s *cliSide) normal(text string) string { + return strings.ReplaceAll(stamp.ReplaceAllString(text, ""), s.home, "") +} + +func TestCLI_every_delivery_command_answers_byte_for_byte_like_python(t *testing.T) { + work := filepath.Join(t.TempDir(), "work") + py, gosd := newSide(t, true, work), newSide(t, false, work) + mustDo(t, os.WriteFile(filepath.Join(work, "out.txt"), []byte("the deliverable"), 0o644)) + realRID := func(s *cliSide) string { + seedOut := strings.TrimSpace(func() string { + cmd := exec.Command("uv", "run", "--no-sync", "python", "-c", "import sqlite3,sys;print(sqlite3.connect(sys.argv[1]).execute('select relationship_id from relationships').fetchone()[0])", filepath.Join(s.state, "relay.sqlite3")) + cmd.Dir = py.dir + cmd.Env = s.env + b, err := cmd.Output() + mustDo(t, err) + return string(b) + }()) + return seedOut + } + pr, gr := realRID(py), realRID(gosd) + if pr != gr { + t.Fatalf("seeded ids differ %s %s", pr, gr) + } + mustDo(t, os.WriteFile(filepath.Join(work, "self.txt"), []byte("declares itself"), 0o644)) + entries, err := store.BuildManifest([]string{filepath.Join(work, "self.txt")}, []string{work}) + mustDo(t, err) + selfHash, err := store.ManifestRevision(entries) + mustDo(t, err) + cases := [][]string{ + {"criteria-show", "--relationship", pr}, + {"criteria-register", "--relationship", pr, "--criterion", "c1=one", "--criterion", "c2=two", "--optional", "c2", "--source-ref", "doc"}, + {"criteria-show", "--relationship", pr}, + {"criteria-register", "--relationship", pr, "--criterion", "c1=one", "--criterion", " c1 =again"}, + {"ack-proof", "--event", "0123456789abcdef0123456789abcdef", "--turn", "t"}, + {"ack-proof", "--event", "nope", "--turn", "t"}, + {"revision-head", "--relationship", pr}, + {"revision-head", "--relationship", "rel-missing"}, + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "failed", "--turn-thread", "01child-task", "--turn-id", "turn-dispatch-1", "--turn-status", "failed"}, + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "failed", "--turn-thread", "01child-task", "--turn-id", "turn-dispatch-1", "--turn-status", "failed"}, + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "ready_for_review", "--turn-thread", "01child-task", "--turn-id", "turn-dispatch-1", "--turn-status", "completed", "--artifact", "/out.txt"}, + // QA: a duplicate emit of the same revision, and a revision declaring itself its own + // predecessor, which is refused with the same reason on both sides. + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "ready_for_review", "--turn-thread", "01child-task", "--turn-id", "turn-dispatch-1", "--turn-status", "completed", "--artifact", "/out.txt"}, + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "ready_for_review", "--turn-thread", "01child-task", "--turn-id", "turn-dispatch-1", "--artifact", "/self.txt", "--supersedes-revision", selfHash}, + {"emit", "--relationship", pr, "--generation", "1", "--outcome", "failed", "--turn-thread", "someone-else", "--turn-id", "turn-dispatch-1", "--turn-status", "failed"}, + {"emit", "--relationship", "rel-missing", "--generation", "1", "--outcome", "failed", "--turn-thread", "a", "--turn-id", "b"}, + {"revision-head", "--relationship", pr}, + {"claim", "--event", "", "--turn", "t1"}, + {"claim", "--event", "", "--turn", "t1"}, + {"ack", "--event", "", "--ack-turn", "t1", "--ack-proof", ""}, + {"ack", "--event", "", "--ack-turn", "t1", "--ack-proof", "wrong"}, + {"verdict", "--event", "", "--verdict", "verified", "--verdict-turn", "v"}, + {"verdict", "--event", "", "--verdict", "verified", "--verdict-turn", "v", "--restoration", ""}, + {"verdict", "--event", "", "--verdict", "needs_changes", "--verdict-turn", "v", "--finding", "c1=needs_changes:fix", "--restoration", "c9"}, + {"deliver"}, + {"deliver", "--event", ""}, + {"reconcile", "--request-id", "del-x"}, + {"recover"}, + {"verify-acks"}, + } + failed := "4a7c8d2e7b0b06e7e2b4b71c55f2b7c1" + for i, args := range cases { + expand := func(s *cliSide) []string { + out := make([]string, len(args)) + for j, a := range args { + a = strings.ReplaceAll(a, "", s.work) + a = strings.ReplaceAll(a, "", failed) + a = strings.ReplaceAll(a, "", AckProof(failed, "t1")) + out[j] = a + } + return out + } + pout, pcode := py.run(expand(py)...) + gout, gcode := gosd.run(expand(gosd)...) + if pcode != gcode || py.normal(pout) != gosd.normal(gout) { + t.Errorf("case %d %v: exit python %d go %d\npython:\n%s\ngo:\n%s", i, args, pcode, gcode, py.normal(pout), gosd.normal(gout)) + } + if args[0] == "emit" && i == 8 { + if m := regexp.MustCompile(`"eventId": "([0-9a-f]{32})"`).FindStringSubmatch(pout); m != nil { + failed = m[1] + } + } + } + if n := strings.Count(fmtCases(cases), "supersedes-revision"); n != 1 { + t.Fatal("the self-supersede case is present") + } +} + +func fmtCases(cases [][]string) string { + var b strings.Builder + for _, c := range cases { + b.WriteString(strings.Join(c, " ")) + } + return b.String() +} diff --git a/internal/relay/delivery/clock.go b/internal/relay/delivery/clock.go new file mode 100644 index 00000000..15967493 --- /dev/null +++ b/internal/relay/delivery/clock.go @@ -0,0 +1,50 @@ +package delivery + +import ( + "math" + "time" +) + +// Clock is the Python clock: seconds as a float, and an isoformat stamp at microseconds. +type Clock interface { + Now() float64 + ISO() string +} + +// ISOOf is datetime.fromtimestamp(now, utc).isoformat(timespec="microseconds"). +func ISOOf(now float64) string { + sec := math.Floor(now) + micro := math.RoundToEven((now - sec) * 1e6) + if micro >= 1e6 { + sec++ + micro -= 1e6 + } + return time.Unix(int64(sec), int64(micro)*1000).UTC().Format("2006-01-02T15:04:05.000000+00:00") +} + +// SystemClock reads the wall clock. +type SystemClock struct{} + +func (SystemClock) Now() float64 { return float64(time.Now().UnixMicro()) / 1e6 } +func (SystemClock) ISO() string { return time.Now().UTC().Format("2006-01-02T15:04:05.000000+00:00") } + +// FakeClock is a clock tests move by hand, starting at Python FakeClock's 1_700_000_000. +type FakeClock struct { + T float64 + // OnISO, when set, runs before every ISO read (tests that make each read differ). + OnISO func() +} + +func NewFakeClock() *FakeClock { return &FakeClock{T: 1_700_000_000} } + +func (c *FakeClock) Now() float64 { return c.T } +func (c *FakeClock) Advance(seconds float64) float64 { + c.T += seconds + return c.T +} +func (c *FakeClock) ISO() string { + if c.OnISO != nil { + c.OnISO() + } + return ISOOf(c.T) +} diff --git a/internal/relay/delivery/criteria.go b/internal/relay/delivery/criteria.go new file mode 100644 index 00000000..c78bb9aa --- /dev/null +++ b/internal/relay/delivery/criteria.go @@ -0,0 +1,487 @@ +package delivery + +import ( + "context" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "slices" + "sort" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Criteria refusals and modes (criteria.py, errors.RefusalReason). +const ( + CriteriaUnregistered = "criteria_unregistered" + CriteriaNotCovered = "criteria_not_covered" + CriteriaSetChanged = "criteria_set_changed" + UnknownCriterion = "unknown_criterion" + FindingsRequired = "findings_required" + DispositionConflict = "disposition_conflict" + ReviewNotBound = "review_not_bound" + Managed = "managed" + Legacy = "legacy" + Covered = "covered" + LegacyUnregistered = "legacy_unregistered" +) + +var dispositions = []string{"verified", "needs_changes", "unverified"} + +// Criterion is one canonical criterion. +type Criterion struct { + ID, Title string + Required bool +} + +func (c Criterion) obj() Obj { + return Obj{{Key: "id", Value: c.ID}, {Key: "title", Value: c.Title}, {Key: "required", Value: c.Required}} +} + +func criteriaObjs(cs []Criterion) []any { + out := make([]any, len(cs)) + for i, c := range cs { + out[i] = c.obj() + } + return out +} + +// jsonCompact is json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False). +func jsonCompact(v any) string { + var b strings.Builder + var write func(any) + write = func(v any) { + switch t := v.(type) { + case Obj: + sorted := append(Obj(nil), t...) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Key < sorted[j].Key }) + b.WriteByte('{') + for i, f := range sorted { + if i > 0 { + b.WriteByte(',') + } + writeRawString(&b, f.Key) + b.WriteByte(':') + write(f.Value) + } + b.WriteByte('}') + case []any: + b.WriteByte('[') + for i, x := range t { + if i > 0 { + b.WriteByte(',') + } + write(x) + } + b.WriteByte(']') + case string: + writeRawString(&b, t) + default: + b.WriteString(dumps(v)) + } + } + write(v) + return b.String() +} + +// writeRawString is a JSON string with ensure_ascii=False: only quote, backslash and controls escape. +func writeRawString(b *strings.Builder, s string) { + b.WriteByte('"') + for _, r := range s { + switch { + case r == '"' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r == '\n': + b.WriteString(`\n`) + case r == '\r': + b.WriteString(`\r`) + case r == '\t': + b.WriteString(`\t`) + case r == '\b': + b.WriteString(`\b`) + case r == '\f': + b.WriteString(`\f`) + case r < 0x20: + fmt.Fprintf(b, `\u%04x`, r) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') +} + +// SetDigest is criteria.set_digest: canonical JSON, so no delimiter can collide two sets. +func SetDigest(cs []Criterion) string { + sorted := slices.Clone(cs) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].ID < sorted[j].ID }) + sum := sha256.Sum256([]byte(jsonCompact(criteriaObjs(sorted)))) + return hex.EncodeToString(sum[:]) +} + +// NormaliseCriteria is normalise_criteria: the one shape a set is stored in. +func NormaliseCriteria(entries []any) ([]Criterion, error) { + var out []Criterion + seen := map[string]bool{} + for _, entry := range entries { + o, ok := entry.(Obj) + if !ok { + return nil, refuse(CriteriaUnregistered, "each criterion is an object with an id and a title") + } + idv, _ := get(o, "id") + titlev, _ := get(o, "title") + id := strings.TrimSpace(pyStrOrEmpty(idv)) + title := strings.TrimSpace(pyStrOrEmpty(titlev)) + if id == "" || title == "" { + return nil, refuse(CriteriaUnregistered, "each criterion needs a non-empty id and title") + } + if seen[id] { + return nil, refuse(CriteriaUnregistered, "duplicate criterion id %s", store.PyRepr(id)) + } + seen[id] = true + required := true + if r, present := get(o, "required"); present { + required = truthy(r) + } + out = append(out, Criterion{id, title, required}) + } + if len(out) == 0 { + return nil, refuse(CriteriaUnregistered, "a criteria set needs at least one criterion") + } + return out, nil +} + +// pyStrOrEmpty is str(value or ""). +func pyStrOrEmpty(v any) string { + if !truthy(v) { + return "" + } + return pyStr(v) +} + +// NormaliseFindings is normalise_findings: criteria and findings merged by id, dispositions in +// the frozen enum, at most one restoration carrier and never an empty one. +func NormaliseFindings(sources ...[]any) ([]any, error) { + var merged []Obj + declared := map[string]bool{} + declaredSet := map[string]bool{} + for _, source := range sources { + for _, item := range source { + o, ok := item.(Obj) + if !ok { + return nil, refuse(DispositionConflict, "each finding is an object") + } + idv, _ := get(o, "id") + id := strings.TrimSpace(pyStrOrEmpty(idv)) + if id == "" { + return nil, refuse(DispositionConflict, "each finding names a criterion id") + } + disposition := "verified" + if v, _ := get(o, "verdict"); truthy(v) { + disposition = pyStr(v) + if _, isText := v.(string); !isText || !slices.Contains(dispositions, disposition) { + return nil, refuse(DispositionConflict, "%s is not one of ('verified', 'needs_changes', 'unverified'); the contract's criteria enum is frozen and a finding outside it cannot be recorded", pyReprValue(v)) + } + } + entry := Obj{{Key: "id", Value: id}, {Key: "verdict", Value: disposition}} + notev, _ := get(o, "note") + if note := strings.TrimSpace(pyStrOrEmpty(notev)); note != "" { + entry = append(entry, F{Key: "note", Value: note}) + } + if flag, present := get(o, "restoration"); present && flag != nil { + b, isBool := flag.(bool) + if !isBool { + return nil, refuse(DispositionConflict, "a finding declares its restoration block with true or false, not %s", pyTypeName(flag)) + } + if declaredSet[id] && declared[id] != b { + return nil, refuse(DispositionConflict, "%s both declares and disclaims the restoration block; one correction carries one block and says so once", store.PyRepr(id)) + } + declared[id], declaredSet[id] = b, true + } + if declared[id] { + entry = append(entry, F{Key: "restoration", Value: true}) + } + merged = slices.DeleteFunc(merged, func(e Obj) bool { return str(e, "id") == id }) + merged = append(merged, entry) + } + } + var carriers []string + for _, e := range merged { + if v, _ := get(e, "restoration"); v == true { + carriers = append(carriers, str(e, "id")) + } + } + if len(carriers) > 1 { + return nil, refuse(DispositionConflict, "%s each declare the restoration block. One correction carries one block, and two candidates is a block nobody can locate", reprList(carriers)) + } + for _, e := range merged { + if v, _ := get(e, "restoration"); v == true { + if _, hasNote := get(e, "note"); !hasNote { + return nil, refuse(DispositionConflict, "%s declares the restoration block and carries no note. The block is the note; a declaration without one names a carrier with nothing in it", store.PyRepr(str(e, "id"))) + } + } + } + out := make([]any, len(merged)) + for i, e := range merged { + out[i] = e + } + return out, nil +} + +// Criteria is criteria.CriteriaService. +type Criteria struct { + Store *store.Store + Clock Clock +} + +func (c *Criteria) registration(rid string, cs []Criterion, digest string, source any) Obj { + return Obj{{Key: "relationshipId", Value: rid}, {Key: "setDigest", Value: digest}, {Key: "sourceRef", Value: source}, {Key: "criteria", Value: criteriaObjs(cs)}, {Key: "mode", Value: Managed}} +} + +// Register is register: an intentional replacement. +func (c *Criteria) Register(ctx context.Context, rid string, entries []any, source any) (Obj, error) { + cs, err := NormaliseCriteria(entries) + if err != nil { + return nil, err + } + digest, now := SetDigest(cs), c.Clock.ISO() + err = c.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { return c.replace(ctx, rid, cs, digest, source, now) }) + if err != nil { + return nil, err + } + return c.registration(rid, cs, digest, source), nil +} + +// EnsureRegistered is ensure_registered: an exact replay is a no-op; a different set refuses. +func (c *Criteria) EnsureRegistered(ctx context.Context, rid string, entries []any, source any) (Obj, error) { + cs, err := NormaliseCriteria(entries) + if err != nil { + return nil, err + } + digest, now := SetDigest(cs), c.Clock.ISO() + err = c.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + existing, err := c.lockedSet(ctx, rid) + if err != nil { + return err + } + if existing == nil { + return c.replace(ctx, rid, cs, digest, source, now) + } + sorted := slices.Clone(cs) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].ID < sorted[j].ID }) + if !slices.Equal(existing.criteria, sorted) || existing.digest != digest || existing.source != source || existing.mode != Managed { + return refuse(CriteriaSetChanged, "criteria for %s are already registered as %s; ensure_registered does not replace them", store.PyRepr(rid), existing.digest) + } + return nil + }) + if err != nil { + return nil, err + } + return c.registration(rid, cs, digest, source), nil +} + +type storedSet struct { + criteria []Criterion + digest string + source any + mode string +} + +func (c *Criteria) lockedSet(ctx context.Context, rid string) (*storedSet, error) { + rows, err := all(ctx, c.Store, "SELECT criterion_id, title, required, source_ref, set_digest, recorded_at FROM canonical_criteria WHERE relationship_id = ? ORDER BY criterion_id", rid) + if err != nil { + return nil, err + } + mode, err := one(ctx, c.Store, "SELECT mode FROM verification_mode WHERE relationship_id = ?", rid) + if err != nil { + return nil, err + } + if len(rows) == 0 && mode == nil { + return nil, nil + } + var cs []Criterion + sources, digests := map[any]bool{}, map[string]bool{} + for _, row := range rows { + if r := row.I("required"); r != 0 && r != 1 { + return nil, refuse(CriteriaSetChanged, "stored required flag for %s is not 0 or 1", store.PyRepr(row.S("criterion_id"))) + } + cs = append(cs, Criterion{row.S("criterion_id"), row.S("title"), row.I("required") == 1}) + sources[row.Opt("source_ref")] = true + digests[row.S("set_digest")] = true + } + if len(cs) == 0 || mode == nil || mode.S("mode") != Managed || len(sources) != 1 || len(digests) != 1 || !digests[SetDigest(cs)] { + return nil, refuse(CriteriaSetChanged, "criteria for %s are stored in a form ensure_registered will not replace or repair", store.PyRepr(rid)) + } + return &storedSet{cs, rows[0].S("set_digest"), rows[0].Opt("source_ref"), mode.S("mode")}, nil +} + +func (c *Criteria) replace(ctx context.Context, rid string, cs []Criterion, digest string, source any, now string) error { + if _, err := execSQL(ctx, c.Store, "DELETE FROM canonical_criteria WHERE relationship_id = ?", rid); err != nil { + return err + } + for _, e := range cs { + required := int64(0) + if e.Required { + required = 1 + } + if _, err := execSQL(ctx, c.Store, "INSERT INTO canonical_criteria (relationship_id, criterion_id, title, required, source_ref, set_digest, recorded_at) VALUES (?,?,?,?,?,?,?)", rid, e.ID, e.Title, required, source, digest, now); err != nil { + return err + } + } + if err := c.writeMode(ctx, rid, Managed, now); err != nil { + return err + } + return journal(ctx, c.Store, "criteria_registered", rid, Obj{{Key: "setDigest", Value: digest}, {Key: "count", Value: int64(len(cs))}}, now) +} + +func (c *Criteria) writeMode(ctx context.Context, rid, mode, now string) error { + _, err := execSQL(ctx, c.Store, "INSERT INTO verification_mode (relationship_id, mode, recorded_at) VALUES (?,?,?) ON CONFLICT(relationship_id) DO UPDATE SET mode = excluded.mode, recorded_at = excluded.recorded_at", rid, mode, now) + return err +} + +// SetMode is set_mode. +func (c *Criteria) SetMode(ctx context.Context, rid, mode string) (Obj, error) { + if mode != Managed && mode != Legacy { + return nil, refuse(DispositionConflict, "unknown verification mode %s", store.PyRepr(mode)) + } + err := c.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { return c.writeMode(ctx, rid, mode, c.Clock.ISO()) }) + return Obj{{Key: "relationshipId", Value: rid}, {Key: "mode", Value: mode}}, err +} + +// Get is get: the registered set, or nil. +func (c *Criteria) Get(ctx context.Context, rid string) (Obj, error) { + rows, err := all(ctx, c.Store, "SELECT * FROM canonical_criteria WHERE relationship_id = ? ORDER BY criterion_id", rid) + if err != nil || len(rows) == 0 { + return nil, err + } + var cs []any + for _, r := range rows { + cs = append(cs, Criterion{r.S("criterion_id"), r.S("title"), r.I("required") != 0}.obj()) + } + return Obj{{Key: "relationshipId", Value: rid}, {Key: "setDigest", Value: rows[0].S("set_digest")}, {Key: "sourceRef", Value: rows[0].Opt("source_ref")}, {Key: "criteria", Value: cs}}, nil +} + +// Mode is mode. +func (c *Criteria) Mode(ctx context.Context, rid string) (string, error) { + row, err := one(ctx, c.Store, "SELECT mode FROM verification_mode WHERE relationship_id = ?", rid) + if err != nil || row == nil { + return Legacy, err + } + return row.S("mode"), nil +} + +// BindReview is bind_review: the set's digest as it stands at review start. +func (c *Criteria) BindReview(ctx context.Context, rid, eventID string) error { + current, err := c.Get(ctx, rid) + if err != nil { + return err + } + var digest any + if current != nil { + digest = str(current, "setDigest") + } + _, err = execSQL(ctx, c.Store, "INSERT OR IGNORE INTO claim_context (event_id, set_digest, bound_at) VALUES (?,?,?)", eventID, digest, c.Clock.ISO()) + return err +} + +// BoundDigest is bound_digest. +func (c *Criteria) BoundDigest(ctx context.Context, eventID string) (any, error) { + row, err := one(ctx, c.Store, "SELECT set_digest FROM claim_context WHERE event_id = ?", eventID) + if err != nil || row == nil { + return nil, err + } + return row.Opt("set_digest"), nil +} + +// Coverage is coverage: the verdict's findings against the canonical set. +func (c *Criteria) Coverage(ctx context.Context, rid, eventID, verdict string, findings []any, reason any, expected any) (Obj, error) { + registered, err := c.Get(ctx, rid) + if err != nil { + return nil, err + } + mode, err := c.Mode(ctx, rid) + if err != nil { + return nil, err + } + if findings == nil { + findings = []any{} + } + if registered == nil { + if mode == Managed { + return nil, refuse(CriteriaUnregistered, "%s is a managed assignment with no canonical criteria; a managed assignment cannot be completed against nothing", store.PyRepr(rid)) + } + return Obj{{Key: "coverage", Value: LegacyUnregistered}, {Key: "setDigest", Value: nil}, {Key: "boundDigest", Value: nil}, {Key: "findings", Value: findings}}, nil + } + digest := str(registered, "setDigest") + bound, err := c.BoundDigest(ctx, eventID) + if err != nil { + return nil, err + } + if mode == Managed && bound == nil && expected == nil && (verdict == "verified" || verdict == "needs_changes") { + return nil, refuse(ReviewNotBound, "this managed review is not bound to a criteria set: claim the event first, or pass the reviewed digest explicitly. Current set is %s", digest) + } + if bound != nil && bound != digest { + return nil, refuse(CriteriaSetChanged, "the criteria set changed after this review was claimed: bound %s, current %s. Findings made against the previous wording cannot certify the current one; claim the review again", pyStr(bound), digest) + } + if expected != nil && expected != digest { + return nil, refuse(CriteriaSetChanged, "expected criteria set %s, but the current set is %s", pyStr(expected), digest) + } + list, _ := get(registered, "criteria") + known, required := map[string]bool{}, []string{} + for _, x := range list.([]any) { + o := x.(Obj) + known[str(o, "id")] = true + if v, _ := get(o, "required"); v == true { + required = append(required, str(o, "id")) + } + } + byID := map[string]Obj{} + for _, f := range findings { + o := f.(Obj) + if !known[str(o, "id")] { + return nil, refuse(UnknownCriterion, "%s is not in this assignment's canonical criteria", store.PyRepr(str(o, "id"))) + } + byID[str(o, "id")] = o + } + hasNote := func(o Obj) bool { v, _ := get(o, "note"); return truthy(v) } + switch verdict { + case "verified": + var missing []string + for _, id := range required { + if o, ok := byID[id]; !ok || str(o, "verdict") != "verified" { + missing = append(missing, id) + } + } + sort.Strings(missing) + if len(missing) > 0 { + return nil, refuse(CriteriaNotCovered, "these required criteria are not recorded as verified: %s", reprList(missing)) + } + case "needs_changes": + ok := false + for _, o := range byID { + if str(o, "verdict") == "needs_changes" && hasNote(o) { + ok = true + } + } + if !ok { + return nil, refuse(FindingsRequired, "a needs_changes verdict needs at least one criterion marked needs_changes with a note; a correction with no findings is one nobody can act on") + } + case "unverified": + ok := false + for _, o := range byID { + if str(o, "verdict") == "unverified" && hasNote(o) { + ok = true + } + } + if !ok { + return nil, refuse(FindingsRequired, "an unverified verdict names at least one criterion it could not verify, with the reason") + } + case "aborted": + if strings.TrimSpace(pyStrOrEmpty(reason)) == "" { + return nil, refuse(FindingsRequired, "an aborted verdict states why") + } + } + return Obj{{Key: "coverage", Value: Covered}, {Key: "setDigest", Value: digest}, {Key: "boundDigest", Value: bound}, {Key: "findings", Value: findings}}, nil +} diff --git a/internal/relay/delivery/criteria_test.go b/internal/relay/delivery/criteria_test.go new file mode 100644 index 00000000..c35a8919 --- /dev/null +++ b/internal/relay/delivery/criteria_test.go @@ -0,0 +1,186 @@ +package delivery + +import ( + "context" + "database/sql" + "path/filepath" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_criteria_registration.py CRR-1..CRR-7. + +const crrSource = "https://linear.app/doc/1" + +func crrSet() []any { + return []any{Obj{{Key: "id", Value: " c2 "}, {Key: "title", Value: " malformed requests are refused "}, {Key: "required", Value: int64(0)}}, Obj{{Key: "id", Value: "c1"}, {Key: "title", Value: "the endpoint returns the agreed shape"}}} +} + +func crit(title string, required any) Obj { + o := Obj{{Key: "title", Value: title}} + if required != nil { + o = append(o, F{Key: "required", Value: required}) + } + return o +} + +func withID(id string, o Obj) Obj { return append(Obj{{Key: "id", Value: id}}, o...) } + +func runCRR(t *testing.T, mode string, goSide func(f *fixture, c *Criteria, out map[string]any)) { + tree := t.TempDir() + python := runPython(t, tree, "crr", mode) + f := newFixture(t, tree) + c := &Criteria{Store: f.store, Clock: f.clock} + out := map[string]any{} + goSide(f, c, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + requireSameTables(t, f, python) +} + +func TestCRR01_an_empty_or_duplicate_set_is_refused_before_any_row(t *testing.T) { + runCRR(t, "refused", func(f *fixture, c *Criteria, out map[string]any) { + var results []any + for _, entries := range [][]any{{}, {withID("c1", crit(" ", nil))}, {withID("c1", crit("one", nil)), "not-an-object"}, {withID("c1", crit("one", nil)), withID(" c1 ", crit("again", nil))}} { + _, err := c.EnsureRegistered(f.ctx, "rel-1", entries, crrSource) + requireReason(t, err, CriteriaUnregistered) + results = append(results, refusalOf(err)) + } + out["r"] = results + out["mode"], _ = c.Mode(f.ctx, "rel-1") + }) +} + +func TestCRR02_the_first_registration_normalises_and_returns_the_managed_set(t *testing.T) { + runCRR(t, "first", func(f *fixture, c *Criteria, out map[string]any) { + set := crrSet() + r, err := c.EnsureRegistered(f.ctx, "rel-1", []any{set[1], set[0]}, crrSource) + mustDo(t, err) + out["r"] = r + }) +} + +func TestCRR03_an_exact_replay_keeps_the_timestamp_and_writes_no_journal(t *testing.T) { + runCRR(t, "replay", func(f *fixture, c *Criteria, out map[string]any) { + _, err := c.EnsureRegistered(f.ctx, "rel-1", crrSet(), crrSource) + mustDo(t, err) + f.clock.Advance(30) + r, err := c.EnsureRegistered(f.ctx, "rel-1", []any{withID("c2", crit("malformed requests are refused", false)), withID("c1", crit(" the endpoint returns the agreed shape ", nil))}, crrSource) + mustDo(t, err) + out["r"] = r + }) +} + +func TestCRR04_a_changed_title_source_or_requirement_refuses_without_rewriting(t *testing.T) { + runCRR(t, "changed", func(f *fixture, c *Criteria, out map[string]any) { + _, err := c.EnsureRegistered(f.ctx, "rel-1", crrSet(), crrSource) + mustDo(t, err) + var results []any + for _, tc := range []struct { + entries []any + source string + }{ + {[]any{withID("c1", crit("the endpoint returns the agreed shape", nil)), withID("c2", crit("a different refusal", false))}, crrSource}, + {crrSet(), "https://linear.app/doc/2"}, + {[]any{withID("c2", crit("malformed requests are refused", true)), withID("c1", crit("the endpoint returns the agreed shape", nil))}, crrSource}, + {append(crrSet(), withID("c3", crit("one more obligation", nil))), crrSource}, + } { + _, err := c.EnsureRegistered(f.ctx, "rel-1", tc.entries, tc.source) + requireReason(t, err, CriteriaSetChanged) + results = append(results, refusalOf(err)) + } + out["r"] = results + }) +} + +func TestCRR05_register_still_replaces_a_set_ensure_would_keep(t *testing.T) { + runCRR(t, "replace", func(f *fixture, c *Criteria, out map[string]any) { + _, err := c.EnsureRegistered(f.ctx, "rel-1", crrSet(), crrSource) + mustDo(t, err) + f.clock.Advance(5) + r, err := c.Register(f.ctx, "rel-1", []any{withID("c9", crit("the replacement obligation", false))}, "operator") + mustDo(t, err) + out["r"] = r + _, err = c.EnsureRegistered(f.ctx, "rel-1", crrSet(), crrSource) + out["r2"] = refusalOf(err) + }) +} + +func TestCRR06_a_corrupt_or_mistyped_stored_set_is_refused_and_left(t *testing.T) { + for _, mode := range []string{"corrupt", "required2"} { + t.Run(mode, func(t *testing.T) { + runCRR(t, mode, func(f *fixture, c *Criteria, out map[string]any) { + normalised := []Criterion{{"c1", "the endpoint returns the agreed shape", true}, {"c2", "malformed requests are refused", false}} + d, now := SetDigest(normalised), f.clock.ISO() + mustDo(t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + rows := [][]any{{"rel-1", "c1", normalised[0].Title, 1, crrSource, d, now}, {"rel-1", "c2", normalised[1].Title, 0, "other-source", "not-the-digest", now}} + mode2 := "legacy" + if mode == "required2" { + rows = [][]any{{"rel-1", "c1", normalised[0].Title, 2, crrSource, d, now}, {"rel-1", "c2", normalised[1].Title, 0, crrSource, d, now}} + mode2 = "managed" + } + for _, r := range rows { + if _, err := execSQL(ctx, f.store, "INSERT INTO canonical_criteria VALUES (?,?,?,?,?,?,?)", r...); err != nil { + return err + } + } + _, err := execSQL(ctx, f.store, "INSERT INTO verification_mode VALUES (?,?,?)", "rel-1", mode2, now) + return err + })) + _, err := c.EnsureRegistered(f.ctx, "rel-1", crrSet(), crrSource) + requireReason(t, err, CriteriaSetChanged) + out["r"] = refusalOf(err) + }) + }) + } +} + +func TestCRR07_two_connections_register_exactly_one_set(t *testing.T) { + for _, same := range []bool{false, true} { + name := map[bool]string{false: "different sets", true: "same set"}[same] + t.Run(name, func(t *testing.T) { + f := newFixture(t, "") + path := filepath.Join(f.tree, "gostate", "relay.sqlite3") + start := make(chan struct{}) + var wg sync.WaitGroup + var mu sync.Mutex + var refusals []string + for i := 0; i < 2; i++ { + entries, source := crrSet(), crrSource + if i == 1 && !same { + entries, source = []any{withID("c9", crit("a rival obligation", nil))}, "rival" + } + s, err := store.Open(f.ctx, path, "") + mustDo(t, err) + t.Cleanup(func() { _ = s.Close() }) + clock := NewFakeClock() + clock.Advance(float64(i * 7)) + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, err := (&Criteria{Store: s, Clock: clock}).EnsureRegistered(context.Background(), "rel-1", entries, source) + if err != nil { + mu.Lock() + refusals = append(refusals, Reason(err)) + mu.Unlock() + } + }() + } + close(start) + wg.Wait() + digests := f.count("SELECT COUNT(DISTINCT set_digest) AS c FROM canonical_criteria") + stamps := f.count("SELECT COUNT(DISTINCT recorded_at) AS c FROM canonical_criteria") + journals := f.count("SELECT COUNT(*) AS c FROM journal WHERE subject = 'rel-1'") + if digests != 1 || stamps != 1 || journals != 1 { + t.Fatalf("digests %d stamps %d journals %d", digests, stamps, journals) + } + if same && len(refusals) != 0 || !same && (len(refusals) != 1 || refusals[0] != CriteriaSetChanged) { + t.Fatalf("refusals %v", refusals) + } + }) + } +} diff --git a/internal/relay/delivery/currency.go b/internal/relay/delivery/currency.go new file mode 100644 index 00000000..30f8b107 --- /dev/null +++ b/internal/relay/delivery/currency.go @@ -0,0 +1,195 @@ +package delivery + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Lineage evidence and currency reasons (currency.py). +const ( + Sole = "sole_revision" + Chain = "declared_chain" + NoRevision = "no_revision" + Fork = "fork" + Cycle = "cycle" + UnknownPredecessor = "unknown_predecessor" + Disconnected = "disconnected" + + StaleGeneration = "stale_generation" + SupersededRevision = "superseded_revision" + RevisionAmbiguous = "revision_ambiguous" +) + +var ambiguousEvidence = []string{Fork, Cycle, UnknownPredecessor, Disconnected} + +func ambiguous(evidence string, nodes []string, detail string) Obj { + competitors := make([]any, len(nodes)) + sorted := slices.Clone(nodes) + slices.Sort(sorted) + for i, n := range sorted { + competitors[i] = n + } + return Obj{{Key: "eventId", Value: nil}, {Key: "revisionHash", Value: nil}, {Key: "evidence", Value: evidence}, {Key: "competitors", Value: competitors}, {Key: "detail", Value: detail}} +} + +// requestedPredecessors is currency._requested_predecessors: only the result whose ruling opened +// this correction is an external root. +func requestedPredecessors(ctx context.Context, s *store.Store, rid string, generation int64) (map[string][]string, error) { + rows, err := all(ctx, s, "SELECT p.event_id, p.revision_hash, v.verdict_turn_id, r.event_id AS request_id, g.dispatch_request_id FROM generations g JOIN verdicts v ON v.next_generation = g.execution_generation JOIN events p ON p.event_id = v.event_id AND p.relationship_id = g.relationship_id JOIN events r ON r.relationship_id = g.relationship_id AND r.execution_generation = g.execution_generation WHERE g.relationship_id = ? AND g.execution_generation = ? AND g.reason = 'needs_changes_revision' AND v.verdict = 'needs_changes' AND p.execution_generation = g.execution_generation - 1 AND p.outcome = ? AND p.stage = 'final' AND p.suppressed_reason IS NULL AND r.outcome = 'revision_request' AND r.producer = 'relay' AND r.stage = 'final' AND r.suppressed_reason IS NULL", rid, generation, "ready_for_review") + if err != nil { + return nil, err + } + anchors := map[string][]string{} + for _, row := range rows { + request, err := store.RevisionRequestEventID(rid, row.S("event_id"), row.S("verdict_turn_id")) + if err != nil { + continue + } + if row.S("request_id") == request && row.S("dispatch_request_id") == "revision-"+request { + anchors[row.S("revision_hash")] = append(anchors[row.S("revision_hash")], row.S("event_id")) + } + } + return anchors, nil +} + +// HeadRevision is currency.head_revision: the one revision this generation stands on, or why not. +func HeadRevision(ctx context.Context, s *store.Store, rid string, generation int64) (Obj, error) { + rows, err := all(ctx, s, "SELECT e.event_id, e.revision_hash, l.supersedes_hash FROM events e LEFT JOIN revision_lineage l ON l.event_id = e.event_id WHERE e.relationship_id = ? AND e.execution_generation = ? AND e.outcome = ? AND e.suppressed_reason IS NULL ORDER BY e.event_id", rid, generation, "ready_for_review") + if err != nil { + return nil, err + } + if len(rows) == 0 { + return Obj{{Key: "eventId", Value: nil}, {Key: "revisionHash", Value: nil}, {Key: "evidence", Value: NoRevision}, {Key: "competitors", Value: []any{}}, {Key: "detail", Value: "no reviewable revision in this generation"}}, nil + } + var nodes []string + hashOf := map[string]string{} + declaredOf := map[string]string{} + byHash := map[string][]string{} + for _, row := range rows { + id := row.S("event_id") + if _, seen := hashOf[id]; !seen { + nodes = append(nodes, id) + } + hashOf[id] = row.S("revision_hash") + declaredOf[id] = row.S("supersedes_hash") + } + for _, id := range nodes { + byHash[hashOf[id]] = append(byHash[hashOf[id]], id) + } + anchors, err := requestedPredecessors(ctx, s, rid, generation) + if err != nil { + return nil, err + } + edges := map[string]string{} + var unresolved []string + for _, id := range nodes { + declared := declaredOf[id] + if declared == "" { + continue + } + targets := append(slices.Clone(byHash[declared]), anchors[declared]...) + if len(targets) != 1 { + unresolved = append(unresolved, id+" -> "+declared) + continue + } + edges[id] = targets[0] + } + if len(unresolved) > 0 { + return ambiguous(UnknownPredecessor, nodes, "a declared predecessor is neither a unique revision of this generation nor its requested correction predecessor: "+strings.Join(unresolved, ", ")), nil + } + for _, start := range nodes { + seen := map[string]bool{start: true} + current := start + for { + next, ok := edges[current] + if !ok { + break + } + current = next + if seen[current] { + return ambiguous(Cycle, nodes, fmt.Sprintf("the declared chain from %s returns to %s", start, current)), nil + } + seen[current] = true + } + } + predecessors := map[string]int{} + for _, id := range nodes { + if target, ok := edges[id]; ok { + predecessors[target]++ + } + } + var forked []string + for target, count := range predecessors { + if count > 1 { + forked = append(forked, target) + } + } + if len(forked) > 0 { + slices.Sort(forked) + return ambiguous(Fork, nodes, "more than one revision declares the same predecessor: "+strings.Join(forked, ", ")), nil + } + targets := map[string]bool{} + for _, t := range edges { + targets[t] = true + } + var tips []string + for _, id := range nodes { + if !targets[id] { + tips = append(tips, id) + } + } + slices.Sort(tips) + if len(tips) != 1 { + return ambiguous(Fork, nodes, fmt.Sprintf("%d revisions in this generation are unsuperseded, so none of them is the head; a revision that replaces another says so when it is emitted", len(tips))), nil + } + tip := tips[0] + covered := map[string]bool{tip: true} + for current := tip; ; { + next, ok := edges[current] + if !ok { + break + } + current = next + covered[current] = true + } + for _, id := range nodes { + if !covered[id] { + return ambiguous(Disconnected, nodes, "the declared chain from the tip does not reach every revision in this generation"), nil + } + } + evidence := Chain + if len(nodes) == 1 && len(edges) == 0 { + evidence = Sole + } + return Obj{{Key: "eventId", Value: tip}, {Key: "revisionHash", Value: hashOf[tip]}, {Key: "evidence", Value: evidence}, {Key: "competitors", Value: []any{}}, {Key: "detail", Value: ""}}, nil +} + +// Currency is currency.currency_of: is this event the thing the assignment stands on now? +func Currency(ctx context.Context, s *store.Store, relationship, event Row) (Obj, error) { + rid := event.S("relationship_id") + if relationship.S("status") != "active" || truthy(relationship.Opt("superseded_by")) { + return Obj{{Key: "current", Value: false}, {Key: "reason", Value: RelationshipNotActive}, {Key: "evidence", Value: nil}, {Key: "headEventId", Value: nil}, {Key: "headRevisionHash", Value: nil}, {Key: "detail", Value: fmt.Sprintf("relationship %s is %s", store.PyRepr(rid), store.PyRepr(relationship.S("status")))}}, nil + } + generation := relationship.I("execution_generation") + if event.I("execution_generation") != generation { + return Obj{{Key: "current", Value: false}, {Key: "reason", Value: StaleGeneration}, {Key: "evidence", Value: nil}, {Key: "headEventId", Value: nil}, {Key: "headRevisionHash", Value: nil}, {Key: "detail", Value: fmt.Sprintf("this event is generation %d and the assignment is on generation %d", event.I("execution_generation"), generation)}}, nil + } + head, err := HeadRevision(ctx, s, rid, generation) + if err != nil { + return nil, err + } + if slices.Contains(ambiguousEvidence, str(head, "evidence")) { + competitors, _ := get(head, "competitors") + return Obj{{Key: "current", Value: false}, {Key: "reason", Value: RevisionAmbiguous}, {Key: "evidence", Value: str(head, "evidence")}, {Key: "headEventId", Value: nil}, {Key: "headRevisionHash", Value: nil}, {Key: "detail", Value: str(head, "detail")}, {Key: "competitors", Value: competitors}}, nil + } + headID, _ := get(head, "eventId") + headHash, _ := get(head, "revisionHash") + if headID != event.S("event_id") { + return Obj{{Key: "current", Value: false}, {Key: "reason", Value: SupersededRevision}, {Key: "evidence", Value: str(head, "evidence")}, {Key: "headEventId", Value: headID}, {Key: "headRevisionHash", Value: headHash}, {Key: "detail", Value: fmt.Sprintf("the current revision of generation %d is %v", generation, headID)}}, nil + } + return Obj{{Key: "current", Value: true}, {Key: "reason", Value: nil}, {Key: "evidence", Value: str(head, "evidence")}, {Key: "headEventId", Value: headID}, {Key: "headRevisionHash", Value: headHash}, {Key: "detail", Value: ""}}, nil +} diff --git a/internal/relay/delivery/currency_ack_test.go b/internal/relay/delivery/currency_ack_test.go new file mode 100644 index 00000000..4f2b3da2 --- /dev/null +++ b/internal/relay/delivery/currency_ack_test.go @@ -0,0 +1,149 @@ +package delivery + +import ( + "testing" +) + +func (v *vcu) dispatched() string { + event := v.queuedEvent(regOpts{recipients: []string{parent, child}}) + v.mustAttempt(event, nil) + v.clock.Advance(5) + return event +} + +func (v *vcu) verifyPending(now *float64) []any { + results, err := v.ack.VerifyPendingAcks(v.ctx, v.host, 8, now) + mustDo(v.t, err) + return results +} + +func runVCUAck(t *testing.T, mode string, goSide func(v *vcu, out map[string]any)) { + tree := t.TempDir() + python := runPython(t, tree, "vcu_ack", mode) + v := newVCU(t, tree) + out := map[string]any{} + goSide(v, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + requireSameTables(t, v.fixture, python) +} + +func TestVCU11_an_offline_ack_is_recorded_intent_and_upgraded_by_a_host(t *testing.T) { + t.Run("without an adapter", func(t *testing.T) { + runVCUAck(t, "offline", func(v *vcu, out map[string]any) { + e := v.dispatched() + rec, err := v.ack.Acknowledge(v.ctx, e, "parent-own-turn", AckProof(e, "parent-own-turn"), true, nil, nil) + mustDo(t, err) + out["ack"] = rec + if str(rec, "_verified") != "unverified_turn" || v.one("SELECT tier FROM ack_evidence WHERE event_id = ?", e).S("tier") != "unverified" { + t.Fatalf("ack %v", rec) + } + }) + }) + t.Run("an unverified ack produces no verdict", func(t *testing.T) { + runVCUAck(t, "no_verdict", func(v *vcu, out map[string]any) { + e := v.dispatched() + rec, err := v.ack.Acknowledge(v.ctx, e, "parent-own-turn", AckProof(e, "parent-own-turn"), true, nil, nil) + mustDo(t, err) + out["ack"] = rec + out["r"] = v.verdict(e, "verified", "v1", nil, nil, nil) + if out["r"].(map[string]any)["reason"] != NotAcknowledged { + t.Fatal("not_acknowledged") + } + }) + }) + t.Run("the dispatch turn alone never verifies", func(t *testing.T) { + runVCUAck(t, "dispatch_turn", func(v *vcu, out map[string]any) { + e := v.dispatched() + turn := v.row(e).S("dispatch_turn_id") + rec, err := v.ack.Acknowledge(v.ctx, e, turn, AckProof(e, turn), true, nil, nil) + mustDo(t, err) + out["ack"] = rec + if str(rec, "_verified") != "unverified_turn" { + t.Fatal("unverified_turn") + } + }) + }) + t.Run("verify_pending_acks upgrades", func(t *testing.T) { + runVCUAck(t, "upgrade", func(v *vcu, out map[string]any) { + e := v.pendingAck() + out["ack"] = v.lastAck + out["r"] = v.verifyPending(nil) + if v.row(e).S("state") != Acknowledged { + t.Fatal("acknowledged") + } + }) + }) +} + +func (v *vcu) pendingAck() string { + e := v.dispatched() + v.host.startTurn(parent, "parent-own-turn", "inProgress", "") + rec, err := v.ack.Acknowledge(v.ctx, e, "parent-own-turn", AckProof(e, "parent-own-turn"), true, nil, nil) + mustDo(v.t, err) + v.lastAck = rec + return e +} + +func TestVCU12_a_deferred_ack_promotion_is_rechecked(t *testing.T) { + t.Run("generation advanced", func(t *testing.T) { + runVCUAck(t, "advanced", func(v *vcu, out map[string]any) { + e := v.pendingAck() + before := v.one("SELECT * FROM acks WHERE event_id = ?", e) + out["ack"] = v.lastAck + v.advance() + out["r"] = v.verifyPending(nil) + after := v.one("SELECT * FROM acks WHERE event_id = ?", e) + if v.row(e).S("state") == Acknowledged || after.S("record") != before.S("record") || after.S("verified") != "unverified_turn" || after.I("accepted") != before.I("accepted") { + t.Fatal("the authored intent is preserved and not promoted") + } + }) + }) + t.Run("paused", func(t *testing.T) { + runVCUAck(t, "paused", func(v *vcu, out map[string]any) { + v.pendingAck() + out["ack"] = v.lastAck + v.setStatusBy("paused", parent) + out["r"] = v.verifyPending(nil) + }) + }) + t.Run("an unchanged refusal is not rejournalled", func(t *testing.T) { + runVCUAck(t, "twice", func(v *vcu, out map[string]any) { + e := v.pendingAck() + out["ack"] = v.lastAck + v.advance() + out["r"] = v.verifyPending(at(v.clock.Now())) + first := v.count("SELECT COUNT(*) AS c FROM journal WHERE subject = ? AND kind = 'ack_verification_withheld'", e) + out["r2"] = v.verifyPending(at(v.clock.Now() + 10000)) + if v.count("SELECT COUNT(*) AS c FROM journal WHERE subject = ? AND kind = 'ack_verification_withheld'", e) != first { + t.Fatal("rejournalled") + } + }) + }) + t.Run("next_check_at", func(t *testing.T) { + runVCUAck(t, "next_check", func(v *vcu, out map[string]any) { + e := v.pendingAck() + out["ack"] = v.lastAck + v.advance() + out["r"] = v.verifyPending(at(100)) + if v.one("SELECT next_check_at FROM ack_evidence WHERE event_id = ?", e).F("next_check_at") <= 100 { + t.Fatal("next_check_at") + } + out["r2"] = v.verifyPending(at(101)) + }) + }) + t.Run("promotes once the blocker clears", func(t *testing.T) { + runVCUAck(t, "cleared", func(v *vcu, out map[string]any) { + e := v.pendingAck() + out["ack"] = v.lastAck + v.setStatusBy("paused", parent) + out["r"] = v.verifyPending(at(100)) + v.setStatusBy("active", parent) + out["r2"] = v.verifyPending(at(100000)) + if v.row(e).S("state") != Acknowledged { + t.Fatal("promoted") + } + }) + }) +} diff --git a/internal/relay/delivery/currency_test.go b/internal/relay/delivery/currency_test.go new file mode 100644 index 00000000..ef0e12cf --- /dev/null +++ b/internal/relay/delivery/currency_test.go @@ -0,0 +1,333 @@ +package delivery + +import ( + "context" + "database/sql" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_verification_currency.py VCU-1..VCU-13 (VCU-5 is python-internal: an inspect.signature +// check; Go's RecordVerdict has no bypass, force or skip parameter by construction). + +type vcu struct { + *fixture + ack *Ack + lastAck Obj + criteria *Criteria +} + +func newVCU(t *testing.T, tree string) *vcu { + f := newFixture(t, tree) + a := NewAck(f.delivery) + return &vcu{fixture: f, ack: a, criteria: a.Criteria} +} + +var criteriaSet = []any{Obj{{Key: "id", Value: "c1"}, {Key: "title", Value: "the endpoint returns the agreed shape"}}, Obj{{Key: "id", Value: "c2"}, {Key: "title", Value: "a malformed request is refused"}}} + +func (v *vcu) acknowledged(text string) string { + if text == "" { + text = "the deliverable" + } + rid := v.register(regOpts{recipients: []string{parent, child}}) + payload := v.readyPayload(rid, 1, []string{v.artifact("out.txt", text)}, 1, assigned("completed")) + _, err := v.accept(payload, store.AcceptOptions{}) + mustDo(v.t, err) + event := str(payload, "eventId") + _, err = v.delivery.Enqueue(v.ctx, event, "", "") + mustDo(v.t, err) + v.mustAttempt(event, nil) + v.clock.Advance(5) + turn := v.host.startTurn(parent, "ack-turn", "inProgress", "") + _, err = v.ack.Acknowledge(v.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, v.host) + mustDo(v.t, err) + return event +} + +func (v *vcu) advance() { + mustDo(v.t, v.store.Transaction(v.ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := OpenGenerationIn(ctx, v.store, v.clock, v.rid, "newer-execution", "needs_changes_revision", "newer-turn") + return err + })) +} + +func (v *vcu) accepted(text string, supersedes *string) Obj { + r, err := LoadRelationship(v.ctx, v.store, v.rid) + mustDo(v.t, err) + payload := v.readyPayload(v.rid, r.Generation, []string{v.artifact("out.txt", text)}, 1, assigned("completed")) + _, err = v.accept(payload, store.AcceptOptions{SupersedesRevision: supersedes}) + mustDo(v.t, err) + return payload +} + +func (v *vcu) revisionHash(event string) string { + return v.one("SELECT revision_hash FROM events WHERE event_id = ?", event).S("revision_hash") +} + +func outcome(record Obj, err error) map[string]any { + if err != nil { + return refusalOf(err) + } + return map[string]any{"ok": record} +} + +func (v *vcu) verdict(event, verdict, turn string, findings []any, reason, expected any) map[string]any { + return outcome(v.ack.RecordVerdict(v.ctx, event, verdict, turn, nil, findings, reason, expected)) +} + +func finding(id, verdict, note string) Obj { + o := Obj{{Key: "id", Value: id}, {Key: "verdict", Value: verdict}} + if note != "" { + o = append(o, F{Key: "note", Value: note}) + } + return o +} + +func (v *vcu) head() Obj { + h, err := HeadRevision(v.ctx, v.store, v.rid, 1) + mustDo(v.t, err) + return h +} + +// run executes one Python scenario of vcu.py and its Go twin, then compares out["r"] and the store. +func runVCU(t *testing.T, mode string, goSide func(v *vcu) any, wantReason string) { + tree := t.TempDir() + python := runPython(t, tree, "vcu", mode) + v := newVCU(t, tree) + got := goSide(v) + requireSameJSON(t, mode, got, python.Out["r"]) + if m, ok := got.(map[string]any); ok && wantReason != "" && m["reason"] != wantReason { + t.Fatalf("%s: want %s, got %v", mode, wantReason, m) + } + requireSameTables(t, v.fixture, python) +} + +func TestVCU01_a_verified_verdict_needs_the_current_head(t *testing.T) { + t.Run("generation advanced", func(t *testing.T) { + runVCU(t, "advanced", func(v *vcu) any { + e := v.acknowledged("") + v.advance() + return v.verdict(e, "verified", "v1", nil, nil, nil) + }, StaleGeneration) + }) + t.Run("two undeclared revisions", func(t *testing.T) { + runVCU(t, "ambiguous", func(v *vcu) any { + e := v.acknowledged("") + v.accepted("a different revision", nil) + return v.verdict(e, "verified", "v1", nil, nil, nil) + }, RevisionAmbiguous) + }) + t.Run("declared successor", func(t *testing.T) { + runVCU(t, "superseded", func(v *vcu) any { + e := v.acknowledged("") + h := v.revisionHash(e) + v.accepted("the corrected revision", &h) + return v.verdict(e, "verified", "v1", nil, nil, nil) + }, SupersededRevision) + }) + t.Run("advance after the caller's preflight read", func(t *testing.T) { + runVCU(t, "advanced", func(v *vcu) any { + e := v.acknowledged("") + r, _ := LoadRelationship(v.ctx, v.store, v.rid) + if r.Generation != 1 { + t.Fatal("preflight reads generation 1") + } + v.advance() + return v.verdict(e, "verified", "v1", nil, nil, nil) + }, StaleGeneration) + }) +} + +func TestVCU02_a_paused_relationship_refuses_every_verdict(t *testing.T) { + runVCU(t, "paused", func(v *vcu) any { + e := v.acknowledged("") + v.setStatusBy("paused", parent) + var out []any + for _, verdict := range []string{"verified", "needs_changes", "unverified", "aborted"} { + r := v.verdict(e, verdict, "v1", nil, "stopping", nil) + if r["reason"] != RelationshipNotActive { + t.Fatalf("%s: %v", verdict, r) + } + out = append(out, r) + } + return out + }, "") +} + +func TestVCU03_needs_changes_on_a_stale_event_is_refused_unverified_is_recorded(t *testing.T) { + t.Run("needs_changes", func(t *testing.T) { + runVCU(t, "nc_stale", func(v *vcu) any { + e := v.acknowledged("") + v.advance() + return v.verdict(e, "needs_changes", "v1", []any{finding("c1", "needs_changes", "still wrong")}, nil, nil) + }, StaleGeneration) + }) + t.Run("unverified", func(t *testing.T) { + runVCU(t, "unv_stale", func(v *vcu) any { + e := v.acknowledged("") + v.advance() + r := v.verdict(e, "unverified", "v1", nil, "could not reach it", nil) + if v.one("SELECT currency FROM verdict_context WHERE event_id = ?", e).S("currency") != StaleGeneration { + t.Fatal("currency stale_generation") + } + return r + }, "") + }) +} + +func TestVCU04_a_replay_returns_the_historical_verdict(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "vcu", "replay") + v := newVCU(t, tree) + e := v.acknowledged("") + first := v.verdict(e, "needs_changes", "v1", []any{finding("c1", "needs_changes", "fix it")}, nil, nil) + again := v.verdict(e, "verified", "v2", nil, nil, nil) + requireSameJSON(t, "first", first, python.Out["first"]) + requireSameJSON(t, "replay", again, python.Out["r"]) + rec := again["ok"].(Obj) + if v, _ := get(rec, "_replay"); v != true || str(rec, "verdict") != "needs_changes" || str(rec, "verdictTurnId") != "v1" { + t.Fatalf("replay %v", rec) + } + requireSameTables(t, v.fixture, python) +} + +func TestVCU06_head_revision_lineage_evidence(t *testing.T) { + for _, tc := range []struct { + mode, evidence string + build func(v *vcu) + }{ + {"sole", Sole, func(v *vcu) { v.acknowledged("") }}, + {"chain", Chain, func(v *vcu) { e := v.acknowledged(""); h := v.revisionHash(e); v.accepted("second revision", &h) }}, + {"fork_undeclared", Fork, func(v *vcu) { v.acknowledged(""); v.accepted("a different revision", nil) }}, + {"fork_shared", Fork, func(v *vcu) { + e := v.acknowledged("") + h := v.revisionHash(e) + v.accepted("branch one", &h) + v.accepted("branch two", &h) + }}, + {"unknown_pred", UnknownPredecessor, func(v *vcu) { + v.acknowledged("") + h := "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + v.accepted("claims to replace something we never saw", &h) + }}, + {"cycle", Cycle, func(v *vcu) { + e := v.acknowledged("") + h := v.revisionHash(e) + s := v.accepted("second revision", &h) + _, err := execSQL(v.ctx, v.store, "UPDATE revision_lineage SET supersedes_hash = ? WHERE event_id = ?", str(s, "revisionHash"), e) + mustDo(v.t, err) + }}, + {"reversed", Fork, func(v *vcu) { v.acknowledged("a different revision"); v.accepted("the deliverable", nil) }}, + } { + t.Run(tc.mode, func(t *testing.T) { + runVCU(t, tc.mode, func(v *vcu) any { + tc.build(v) + h := v.head() + if str(h, "evidence") != tc.evidence { + t.Fatalf("head %v", h) + } + return h + }, "") + }) + } +} + +func TestVCU07_a_revision_cannot_declare_itself(t *testing.T) { + runVCU(t, "self", func(v *vcu) any { + rid := v.register(regOpts{}) + payload := v.readyPayload(rid, 1, []string{v.artifact("out.txt", "self referential")}, 1, assigned("completed")) + h := str(payload, "revisionHash") + _, err := v.accept(payload, store.AcceptOptions{SupersedesRevision: &h}) + return refusalOf(err) + }, store.ReasonRevisionLineageInvalid) +} + +func TestVCU08_criteria_coverage_on_verdicts(t *testing.T) { + both := []any{finding("c1", "verified", ""), finding("c2", "verified", "")} + claimed := func(v *vcu) string { + e := v.acknowledged("") + _, err := v.criteria.Register(v.ctx, v.rid, criteriaSet, "https://linear.app/doc/1") + mustDo(v.t, err) + _, err = v.ack.ClaimVerification(v.ctx, e, "ack-turn") + mustDo(v.t, err) + return e + } + for _, tc := range []struct { + mode, reason string + run func(v *vcu) any + }{ + {"managed_none", CriteriaUnregistered, func(v *vcu) any { + e := v.acknowledged("") + _, err := v.criteria.SetMode(v.ctx, v.rid, Managed) + mustDo(v.t, err) + return v.verdict(e, "verified", "v1", nil, nil, nil) + }}, + {"legacy", "", func(v *vcu) any { return v.verdict(v.acknowledged(""), "verified", "v1", nil, nil, nil) }}, + {"missing", CriteriaNotCovered, func(v *vcu) any { + return v.verdict(claimed(v), "verified", "v1", []any{finding("c1", "verified", "")}, nil, nil) + }}, + {"covered", "", func(v *vcu) any { return v.verdict(claimed(v), "verified", "v1", both, nil, nil) }}, + {"no_note", FindingsRequired, func(v *vcu) any { + return v.verdict(claimed(v), "needs_changes", "v1", []any{finding("c1", "needs_changes", "")}, nil, nil) + }}, + {"unknown_id", UnknownCriterion, func(v *vcu) any { + return v.verdict(claimed(v), "verified", "v1", []any{finding("nope", "verified", "")}, nil, nil) + }}, + {"bad_disposition", DispositionConflict, func(v *vcu) any { + return v.verdict(claimed(v), "verified", "v1", []any{finding("c1", "regressed", "")}, nil, nil) + }}, + } { + t.Run(tc.mode, func(t *testing.T) { runVCU(t, tc.mode, tc.run, tc.reason) }) + } +} + +func TestVCU09_criteria_currency_binds_the_review(t *testing.T) { + both := []any{finding("c1", "verified", ""), finding("c2", "verified", "")} + for _, tc := range []struct { + mode, reason string + run func(v *vcu) any + }{ + {"edited", CriteriaSetChanged, func(v *vcu) any { + e := v.acknowledged("") + _, err := v.criteria.Register(v.ctx, v.rid, criteriaSet, "https://linear.app/doc/1") + mustDo(v.t, err) + _, err = v.ack.ClaimVerification(v.ctx, e, "ack-turn") + mustDo(v.t, err) + _, err = v.criteria.Register(v.ctx, v.rid, []any{Obj{{Key: "id", Value: "c1"}, {Key: "title", Value: "the endpoint returns a COMPLETELY different shape"}}, Obj{{Key: "id", Value: "c2"}, {Key: "title", Value: "a malformed request is refused"}}}, "https://linear.app/doc/1") + mustDo(v.t, err) + return v.verdict(e, "verified", "v1", both, nil, nil) + }}, + {"wrong_digest", CriteriaSetChanged, func(v *vcu) any { + e := v.acknowledged("") + _, err := v.criteria.Register(v.ctx, v.rid, criteriaSet, "https://linear.app/doc/1") + mustDo(v.t, err) + _, err = v.ack.ClaimVerification(v.ctx, e, "ack-turn") + mustDo(v.t, err) + return v.verdict(e, "verified", "v1", both, nil, "0000000000000000000000000000000000000000000000000000000000000000") + }}, + {"no_claim", ReviewNotBound, func(v *vcu) any { + e := v.acknowledged("") + _, err := v.criteria.Register(v.ctx, v.rid, criteriaSet, "https://linear.app/doc/1") + mustDo(v.t, err) + return v.verdict(e, "verified", "v1", both, nil, nil) + }}, + {"explicit", "", func(v *vcu) any { + e := v.acknowledged("") + reg, err := v.criteria.Register(v.ctx, v.rid, criteriaSet, "https://linear.app/doc/1") + mustDo(v.t, err) + return v.verdict(e, "verified", "v1", both, nil, str(reg, "setDigest")) + }}, + } { + t.Run(tc.mode, func(t *testing.T) { runVCU(t, tc.mode, tc.run, tc.reason) }) + } +} + +func TestVCU10_the_set_digest_resists_delimiter_injection(t *testing.T) { + a := SetDigest([]Criterion{{"a", "b|c", true}}) + b := SetDigest([]Criterion{{"a|b", "c", true}}) + python := pythonValue(t, "from codex_session_relay.criteria import set_digest; print(set_digest([{'id': 'a', 'title': 'b|c', 'required': True}]), set_digest([{'id': 'a|b', 'title': 'c', 'required': True}]))") + if a == b || python != a+" "+b { + t.Fatalf("go %s %s python %s", a, b, python) + } +} diff --git a/internal/relay/delivery/delivery_a_test.go b/internal/relay/delivery/delivery_a_test.go new file mode 100644 index 00000000..a5137cd1 --- /dev/null +++ b/internal/relay/delivery/delivery_a_test.go @@ -0,0 +1,266 @@ +package delivery + +import ( + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_delivery.py properties DEL-1..DEL-10. Each Go run shares its fixture tree with the real +// Python run of the same scenario, so paths, hashes and ids agree, and the whole store, the +// returned records and the refusals are compared with what Python wrote. + +func TestDEL01_an_accepted_final_event_is_queued_idempotently_and_eligible(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del01") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + first := f.row(event) + again, err := f.delivery.Enqueue(f.ctx, event, "", "") + mustDo(t, err) + requireSameJSON(t, "first", first, python.Out["first"]) + requireSameJSON(t, "again", again, python.Out["again"]) + requireSameJSON(t, "eligible", f.eligible(), python.Out["eligible"]) + if f.count("SELECT COUNT(*) AS c FROM deliveries") != 1 || first.S("state") != Queued || first.S("recipient_task_id") != parent { + t.Fatalf("one queued delivery to the parent: %v", first) + } + requireSameTables(t, f, python) +} + +func TestDEL02_a_staged_event_cannot_be_queued(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del02") + f := newFixture(t, tree) + rid := f.register(regOpts{}) + payload := f.readyPayload(rid, 1, []string{f.artifact("out.txt", "still working")}, 1, assigned("inProgress")) + _, err := f.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + _, err = f.delivery.Enqueue(f.ctx, str(payload, "eventId"), "", "") + requireReason(t, err, NotClaimable) + requireSameJSON(t, "refusal", refusalOf(err), python.Out["refused"]) + requireSameTables(t, f, python) +} + +func TestDEL03_a_recipient_outside_the_authorized_scope_is_refused_before_any_transport(t *testing.T) { + t.Run("out-of-scope recipient at enqueue", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del03", "scope") + f := newFixture(t, tree) + event := f.readyEvent(regOpts{}) + _, err := f.delivery.Enqueue(f.ctx, event, "", "somebody-else") + requireReason(t, err, RecipientNotAuthorized) + requireSameJSON(t, "refusal", refusalOf(err), python.Out["refused"]) + if len(f.host.sends) != 0 { + t.Fatal("the transport was called") + } + requireSameTables(t, f, python) + }) + t.Run("another assignment's parent at enqueue", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del03", "other") + f := newFixture(t, tree) + f.otherAssignment() + event := f.readyEvent(regOpts{recipients: []string{parent, "01other-parent"}}) + _, err := f.delivery.Enqueue(f.ctx, event, "", "01other-parent") + requireReason(t, err, RecipientNotAuthorized) + if !strings.Contains(Detail(err), "its own parent") { + t.Fatalf("detail %q", Detail(err)) + } + requireSameJSON(t, "refusal", refusalOf(err), python.Out["refused"]) + row, _ := f.delivery.Find(f.ctx, event) + if row != nil || python.Out["row"] != nil { + t.Fatalf("a row was created: go %v python %v", row, python.Out["row"]) + } + }) + t.Run("tampered row at attempt", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del03", "tampered") + f := newFixture(t, tree) + f.otherAssignment() + event := f.queuedEvent(regOpts{recipients: []string{parent, "01other-parent"}}) + f.host.addThread("01other-parent") + _, err := execSQL(f.ctx, f.store, "UPDATE deliveries SET recipient_task_id = ?, recipient_thread_id = ? WHERE event_id = ?", "01other-parent", "01other-parent", event) + mustDo(t, err) + _, err = f.attempt(event, nil) + requireReason(t, err, RecipientNotAuthorized) + requireSameJSON(t, "refusal", refusalOf(err), python.Out["refused"]) + if len(f.host.sends) != 0 || len(python.Sends) != 0 { + t.Fatal("nothing may reach the host") + } + }) + t.Run("own parent control", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del03", "own") + f := newFixture(t, tree) + f.otherAssignment() + event := f.queuedEvent(regOpts{}) + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != Dispatched || f.row(event).S("recipient_task_id") != parent { + t.Fatalf("record %v", record) + } + }) +} + +func TestDEL04_the_message_carries_the_event_never_a_recipient_turn_or_override(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del04") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + preview, err := f.delivery.PreviewMessage(f.ctx, event) + mustDo(t, err) + again, _ := f.delivery.PreviewMessage(f.ctx, event) + if preview != again || preview != python.Out["preview"] { + t.Fatalf("preview differs from Python:\n%s\n--\n%v", preview, python.Out["preview"]) + } + if !strings.Contains(preview, event) || strings.Contains(strings.ReplaceAll(preview, "your own turn id", ""), "turn-") { + t.Fatal("the message must carry the event and no recipient turn id") + } + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + requireSameJSON(t, "sends", sendsJSON(f.host), python.Sends) + sent := f.host.sends[0] + if sent.thread != parent { + t.Fatalf("sent to %s", sent.thread) + } + for _, forbidden := range []string{"model", "effort", "sandbox", "approvalPolicy", "reasoning"} { + if strings.Contains(sent.message, forbidden) { + t.Fatalf("message carries %q", forbidden) + } + } + requireSameTables(t, f, python) +} + +func TestDEL05_an_active_recipient_is_deferred_without_an_attempt_or_interruption(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del05") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + turn := f.host.startTurn(parent, "", "inProgress", "") + f.host.threads[parent].status = "active" + record := f.mustAttempt(event, nil) + if record != nil || python.Out["record"] != nil { + t.Fatalf("a busy recipient returns None: go %v python %v", record, python.Out["record"]) + } + still, _ := f.host.ReadTurn(parent, turn.TurnID) + if still.Status != "inProgress" || python.Out["turn"] != "inProgress" { + t.Fatal("the running turn was interrupted") + } + if f.row(event).S("state") != DeferredBusy || f.count("SELECT COUNT(*) AS c FROM attempts") != 0 || len(f.host.sends) != 0 { + t.Fatal("deferred_busy with no attempt and no send") + } + requireSameTables(t, f, python) +} + +func TestDEL06_a_transport_busy_refusal_produces_a_real_deferred_attempt(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del06") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"busy"} + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != DeferredBusy || str(record, "sendAttempted") != "no" || str(record, "failedOperation") != "thread/read" { + t.Fatalf("record %v", record) + } + if v, _ := get(record, "retrySafe"); v != true { + t.Fatal("retrySafe") + } + requireSameTables(t, f, python) +} + +func TestDEL07_an_idle_recipient_gets_a_real_turn_and_a_steer_is_recorded(t *testing.T) { + t.Run("idle -> fresh turn", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del07") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != Dispatched || str(record, "turnId") == "" || f.row(event).S("state") != Dispatched { + t.Fatalf("record %v", record) + } + requireSameTables(t, f, python) + }) + t.Run("running turn -> steered_observed_turn", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del07", "steer") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.startTurn(parent, "already-running", "inProgress", "") + f.host.script = []string{"steer_existing"} + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "turnId") != "already-running" || str(record, "_turnOrigin") != "steered_observed_turn" { + t.Fatalf("record %v", record) + } + if v, _ := get(record, "_turnPreviouslyObserved"); v != true { + t.Fatal("_turnPreviouslyObserved") + } + requireSameTables(t, f, python) + }) +} + +func TestDEL08_dispatched_is_not_delivered(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del08") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.mustAttempt(event, nil) + item, err := f.delivery.SnapshotItem(f.ctx, event) + mustDo(t, err) + got := Obj{} + for _, k := range []string{"state", "reported", "acknowledged", "phase"} { + v, _ := get(item, k) + got = append(got, F{Key: k, Value: v}) + } + requireSameJSON(t, "snapshot", got, python.Out["snapshot"]) + if str(item, "reported") != "dispatched_awaiting_ack" { + t.Fatalf("snapshot %v", item) + } +} + +func TestDEL09_request_id_is_distinct_from_the_event_id(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del07") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + record := f.mustAttempt(event, nil) + request := str(record, "requestId") + want, _ := python.Out["record"].(map[string]any) + if request == event || !strings.HasPrefix(request, "del-") || !strings.Contains(request, event[:12]) || request != want["requestId"] { + t.Fatalf("request id %q (python %v)", request, want["requestId"]) + } +} + +func TestDEL10_an_unsupported_approval_policy_is_stored_not_woken_and_held(t *testing.T) { + for _, policy := range []string{"on-request", "untrusted"} { + t.Run(policy, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del10", policy) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.threads[parent].approvalPolicy = policy + f.host.script = []string{"approval_policy"} + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != InboxOnly || str(record, "recipientApprovalPolicy") != policy { + t.Fatalf("record %v", record) + } + item, err := f.delivery.SnapshotItem(f.ctx, event) + mustDo(t, err) + got := Obj{} + for _, k := range []string{"state", "reported", "holdReason", "phase"} { + v, _ := get(item, k) + got = append(got, F{Key: k, Value: v}) + } + requireSameJSON(t, "snapshot", got, python.Out["snapshot"]) + f.clock.Advance(100000) + if len(f.eligible()) != 0 { + t.Fatal("an inbox-only delivery is never retried") + } + requireSameTables(t, f, python) + }) + } +} diff --git a/internal/relay/delivery/delivery_b_test.go b/internal/relay/delivery/delivery_b_test.go new file mode 100644 index 00000000..067230ee --- /dev/null +++ b/internal/relay/delivery/delivery_b_test.go @@ -0,0 +1,332 @@ +package delivery + +import ( + "math" + "strings" + "testing" +) + +// test_delivery.py properties DEL-11..DEL-20. + +func TestDEL11_each_retry_opens_a_new_attempt_and_never_replays_the_first_request(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del11") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"busy"} + first := f.mustAttempt(event, nil) + f.clock.Advance(3600) + second := f.mustAttempt(event, at(f.clock.Now())) + requireSameJSON(t, "first", first, python.Out["first"]) + requireSameJSON(t, "second", second, python.Out["second"]) + if str(first, "requestId") == str(second, "requestId") || str(second, "deliveryState") != Dispatched { + t.Fatalf("records %v %v", first, second) + } + replayed := 0 + for _, s := range f.host.sends { + if s.requestID == str(first, "requestId") { + replayed++ + } + } + if replayed != 1 { + t.Fatalf("the first request id was sent %d times", replayed) + } + requireSameTables(t, f, python) +} + +func TestDEL12_a_dispatched_or_uncertain_delivery_is_never_claimed_again(t *testing.T) { + t.Run("dispatched", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del12", "dispatched") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.mustAttempt(event, nil) + f.clock.Advance(100000) + if again := f.mustAttempt(event, at(f.clock.Now())); again != nil || python.Out["again"] != nil { + t.Fatalf("claimed again: %v", again) + } + requireSameTables(t, f, python) + }) + t.Run("held_uncertain after 5 clock advances", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del12", "uncertain") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"turn_start_fail"} + first := f.mustAttempt(event, nil) + requireSameJSON(t, "first", first, python.Out["first"]) + for i := 0; i < 5; i++ { + f.clock.Advance(86400) + if len(f.eligible()) != 0 || f.mustAttempt(event, at(f.clock.Now())) != nil { + t.Fatal("an uncertain attempt was retried by elapsed time") + } + } + if f.row(event).S("state") != HeldUncertain || f.count("SELECT COUNT(*) AS c FROM attempts") != 1 { + t.Fatal("one attempt, still held_uncertain") + } + requireSameTables(t, f, python) + }) +} + +func TestDEL13_flood_bounds_cap_attempts_and_pace_sends(t *testing.T) { + t.Run("direct pre-send failures -> attempt_cap", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del13", "direct") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + var records []any + for i := int64(0); i < f.delivery.Policy.MaxAttempts; i++ { + f.host.script = []string{"read_fail"} + f.clock.Advance(100000) + records = append(records, f.mustAttempt(event, at(f.clock.Now()))) + } + requireSameJSON(t, "records", records, python.Out["records"]) + if f.row(event).S("hold_reason") != AttemptCap { + t.Fatal("attempt_cap") + } + f.clock.Advance(100000) + if len(f.eligible()) != 0 { + t.Fatal("a capped delivery is not eligible") + } + requireSameTables(t, f, python) + }) + t.Run("reconciled pre-send failures -> attempt_cap", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del13", "reconciled") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + rc := NewReconciler(f.delivery) + var outcomes []any + for i := int64(0); i < f.delivery.Policy.MaxAttempts; i++ { + f.host.script = []string{"in_progress"} + f.clock.Advance(100000) + record := f.mustAttempt(event, at(f.clock.Now())) + if record == nil { + break + } + request := str(record, "requestId") + f.host.ledger[request] = Obj{{Key: "requestId", Value: request}, {Key: "status", Value: "failed"}, {Key: "error", Value: "thread/read: refused"}, {Key: "rpcError", Value: Obj{{Key: "code", Value: "internal"}, {Key: "message", Value: "refused"}}}} + outcome, err := rc.ReconcileAttempt(f.ctx, request, f.host, at(f.clock.Now())) + mustDo(t, err) + outcomes = append(outcomes, outcome) + } + requireSameJSON(t, "outcomes", outcomes, python.Out["outcomes"]) + if f.row(event).S("hold_reason") != AttemptCap || f.count("SELECT COUNT(*) AS c FROM attempts") > f.delivery.Policy.MaxAttempts { + t.Fatal("the reconciled path obeys the cap") + } + requireSameTables(t, f, python) + }) + t.Run("min interval", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del13", "interval") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"read_fail"} + f.mustAttempt(event, nil) + mustDo(t, f.delivery.Reschedule(f.ctx, event, WithheldPreSend, f.clock.Now(), f.row(event).I("attempt_count"))) + if again := f.mustAttempt(event, at(f.clock.Now()+1)); again != nil || python.Out["again"] != nil { + t.Fatal("the minimum interval refuses the send") + } + requireSameTables(t, f, python) + }) + t.Run("hourly cap", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del13", "hourly") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + now := f.clock.Now() + _, err := execSQL(f.ctx, f.store, "INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,?,?)", parent, math.Floor(now/3600)*3600, f.delivery.Policy.MaxSendsPerRecipientPerHour, 0) + mustDo(t, err) + if again := f.mustAttempt(event, at(now)); again != nil || len(f.host.sends) != 0 { + t.Fatal("the hourly cap refuses the send") + } + requireSameTables(t, f, python) + }) +} + +func lifecycleRow(f *fixture) Row { + return f.one("SELECT * FROM recipient_lifecycle WHERE task_id = ?", parent) +} + +func TestDEL14_host_lifecycle_withholds_are_deferrals_not_holds(t *testing.T) { + for _, tc := range []struct { + mode, reason string + apply func(*fakeThread, *fakeHost) + }{ + {"archived", RecipientArchived, func(th *fakeThread, _ *fakeHost) { th.archived = boolp(true) }}, + {"paused", RecipientPaused, func(th *fakeThread, _ *fakeHost) { th.goalStatus = "paused" }}, + {"budget", RecipientBudgetLimited, func(th *fakeThread, _ *fakeHost) { th.goalStatus = "budgetLimited" }}, + {"noinput", RecipientCannotAccept, func(th *fakeThread, _ *fakeHost) { th.canAcceptInput = false }}, + {"idle", RecipientArchived, func(th *fakeThread, _ *fakeHost) { th.archived = boolp(true); th.status = "idle" }}, + } { + t.Run(tc.mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del14", tc.mode) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + tc.apply(f.host.threads[parent], f.host) + record := f.mustAttempt(event, nil) + if record != nil || python.Out["record"] != nil || python.Out["status"] != "active" { + t.Fatalf("withheld returns None: %v", record) + } + row := f.row(event) + if !row.N("hold_reason") || row.N("next_eligible_at") || lifecycleRow(f).S("withhold_reason") != tc.reason || len(f.host.sends) != 0 { + t.Fatalf("deferral: %v", row) + } + requireSameTables(t, f, python) + }) + } +} + +func TestDEL15_an_unreadable_lifecycle_withholds_rather_than_guessing(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del14", "unreadable") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.readFailures["read_goal_status"] = true + if record := f.mustAttempt(event, nil); record != nil { + t.Fatal("withheld") + } + observed := lifecycleRow(f) + if observed.S("deliverable") != "unknown" || observed.S("withhold_reason") != LifecycleUnknown || len(f.host.sends) != 0 { + t.Fatalf("observed %v", observed) + } + requireSameTables(t, f, python) +} + +func TestDEL16_a_later_good_observation_releases_the_withheld_delivery(t *testing.T) { + for _, mode := range []string{"paused", "archived"} { + t.Run(mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del16", mode) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + th := f.host.threads[parent] + if mode == "paused" { + th.goalStatus = "paused" + f.mustAttempt(event, nil) + th.goalStatus = "active" + } else { + th.archived = boolp(true) + f.mustAttempt(event, nil) + th.archived = boolp(false) + } + f.clock.Advance(f.delivery.Policy.LifecycleRecheck + 1) + requireSameJSON(t, "eligible", f.eligible(), python.Out["eligible"]) + record := f.mustAttempt(event, at(f.clock.Now())) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != Dispatched { + t.Fatal("released and dispatched") + } + requireSameTables(t, f, python) + }) + } +} + +func TestDEL17_a_deactivation_between_precheck_and_claim_blocks_the_send(t *testing.T) { + for _, mode := range []string{"paused", "archived", "supersede", "before"} { + t.Run(mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del17", mode) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + if mode == "before" { + f.setStatus("paused") + if len(f.eligible()) != 0 { + t.Fatal("a paused relationship is never eligible") + } + requireSameTables(t, f, python) + return + } + f.host.onGoalRead = func(string) { + f.host.onGoalRead = nil + if mode == "supersede" { + f.supersede("rel-bbbbbbbbbbbbbbbb") + } else { + f.setStatus(mode) + } + } + if record := f.mustAttempt(event, nil); record != nil || python.Out["record"] != nil { + t.Fatalf("record %v", record) + } + if len(f.host.sends) != 0 || f.count("SELECT COUNT(*) AS c FROM attempts") != 0 { + t.Fatal("no send and no attempt") + } + requireSameTables(t, f, python) + }) + } +} + +func TestDEL18_a_deactivated_assignment_is_withheld_with_a_returned_record(t *testing.T) { + for _, status := range []string{"cancelled", "paused", "archived"} { + t.Run(status, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del18", status) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.setStatus(status) + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "withheldReason") != RelationshipNotActive || str(record, "relationshipStatus") != status || len(f.host.sends) != 0 { + t.Fatalf("record %v", record) + } + row := f.row(event) + if row.S("state") != WithheldPreSend || !row.N("hold_reason") || f.count("SELECT COUNT(*) AS c FROM attempts") != 0 { + t.Fatal("withheld, no permanent hold, no attempt") + } + entry := f.one("SELECT * FROM journal WHERE kind = ? AND subject = ?", "delivery_withheld_inactive", event) + if entry == nil || !strings.Contains(entry.S("detail"), status) { + t.Fatal("the deactivation is journalled with its status") + } + requireSameTables(t, f, python) + }) + } +} + +func TestDEL19_a_stopped_assignment_reads_nothing_from_the_host(t *testing.T) { + f := newFixture(t, "") + event := f.queuedEvent(regOpts{}) + f.setStatus("cancelled") + stopped := &counted{fakeHost: f.host} + _, err := f.delivery.Attempt(f.ctx, event, stopped, at(f.clock.Now()), "") + mustDo(t, err) + if len(stopped.calls) != 0 || lifecycleRow(f) != nil { + t.Fatalf("the host was read for a stopped assignment: %v", stopped.calls) + } + g := newFixture(t, "") + active := g.queuedEvent(regOpts{}) + control := &counted{fakeHost: g.host} + _, err = g.delivery.Attempt(g.ctx, active, control, at(g.clock.Now()), "") + mustDo(t, err) + if !strings.Contains(strings.Join(control.calls, ","), "read_thread") { + t.Fatal("the control must see the reads an active assignment makes") + } +} + +func TestDEL20_resuming_delivers_the_same_event_once_and_an_active_one_is_untouched(t *testing.T) { + t.Run("resume", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del20", "resume") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.setStatus("paused") + f.mustAttempt(event, nil) + f.resume() + f.clock.Advance(f.delivery.Policy.LifecycleRecheck + 1) + record := f.mustAttempt(event, at(f.clock.Now())) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "deliveryState") != Dispatched || len(f.host.sends) != 1 { + t.Fatal("delivered once") + } + requireSameTables(t, f, python) + }) + t.Run("active", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del20", "active") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + requireSameTables(t, f, python) + }) +} diff --git a/internal/relay/delivery/delivery_c_test.go b/internal/relay/delivery/delivery_c_test.go new file mode 100644 index 00000000..fd7d2a55 --- /dev/null +++ b/internal/relay/delivery/delivery_c_test.go @@ -0,0 +1,307 @@ +package delivery + +import ( + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_delivery.py properties DEL-21..DEL-30. + +func TestDEL21_a_deactivation_is_reported_during_a_backoff_and_never_shortens_it(t *testing.T) { + t.Run("reported while a busy backoff runs, never shortened", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del21", "running") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.threads[parent].status = "active" + if first := f.mustAttempt(event, nil); first != nil { + t.Fatal("deferred busy") + } + deferred := f.row(event).F("next_eligible_at") + f.setStatus("cancelled") + record := f.mustAttempt(event, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + if str(record, "withheldReason") != RelationshipNotActive || f.row(event).F("next_eligible_at") < deferred { + t.Fatalf("record %v", record) + } + requireSameJSON(t, "after", f.row(event).F("next_eligible_at"), python.Out["after"]) + requireSameTables(t, f, python) + }) + t.Run("a backoff extended after the row was read still wins", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del21", "extended") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.setStatus("cancelled") + stale, err := LoadRelationship(f.ctx, f.store, f.rid) + mustDo(t, err) + far := f.clock.Now() + 100000 + _, err = execSQL(f.ctx, f.store, "UPDATE deliveries SET next_eligible_at = ? WHERE event_id = ?", far, event) + mustDo(t, err) + record, err := f.delivery.WithholdInactive(f.ctx, event, stale, f.clock.Now(), 0) + mustDo(t, err) + requireSameJSON(t, "record", record, python.Out["record"]) + if f.row(event).F("next_eligible_at") != far { + t.Fatal("the later deadline survives") + } + requireSameTables(t, f, python) + }) +} + +func TestDEL22_a_superseded_relationship_is_left_to_the_supersession_path(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del22", "superseded") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.supersede("rel-bbbbbbbbbbbbbbbb") + if record := f.mustAttempt(event, nil); record != nil || len(f.host.sends) != 0 { + t.Fatalf("record %v", record) + } + if f.one("SELECT * FROM journal WHERE kind = ?", "delivery_withheld_inactive") != nil { + t.Fatal("never tagged relationship_not_active") + } + requireSameTables(t, f, python) +} + +func TestDEL23_a_resume_racing_the_withhold_leaves_the_delivery_alone(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del22", "race") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + stale, err := LoadRelationship(f.ctx, f.store, f.rid) + mustDo(t, err) + stale.Status = "paused" + record, err := f.delivery.WithholdInactive(f.ctx, event, stale, f.clock.Now(), 0) + mustDo(t, err) + if record != nil || python.Out["record"] != nil || f.row(event).S("state") != Queued || f.one("SELECT * FROM journal WHERE kind = ?", "delivery_withheld_inactive") != nil { + t.Fatal("a stale reading holds nothing and journals nothing") + } + requireSameTables(t, f, python) +} + +func TestDEL24_guarded_transitions_prevent_duplicate_sends(t *testing.T) { + t.Run("a stale busy observation cannot overwrite a dispatch", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del24", "busy") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.mustAttempt(event, nil) + stale := Row{} + for k, v := range f.row(event) { + stale[k] = v + } + stale["state"], stale["attempt_count"] = Queued, int64(0) + mustDo(t, f.delivery.DeferBusy(f.ctx, event, stale, f.clock.Now())) + if f.row(event).S("state") != Dispatched { + t.Fatal("the dispatch stands") + } + f.clock.Advance(100000) + if again := f.mustAttempt(event, at(f.clock.Now())); again != nil || len(f.host.sends) != 1 { + t.Fatal("sent once") + } + requireSameTables(t, f, python) + }) + t.Run("reconciling an older attempt cannot reopen a dispatch", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del24", "reconcile") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"busy"} + first := f.mustAttempt(event, nil) + f.clock.Advance(3600) + second := f.mustAttempt(event, at(f.clock.Now())) + requireSameJSON(t, "second", second, python.Out["second"]) + outcome, err := NewReconciler(f.delivery).ReconcileAttempt(f.ctx, str(first, "requestId"), f.host, nil) + mustDo(t, err) + requireSameJSON(t, "reconciled", outcome, python.Out["reconciled"]) + f.clock.Advance(100000) + if again := f.mustAttempt(event, at(f.clock.Now())); again != nil || f.row(event).S("state") != Dispatched || len(f.host.sends) != 2 { + t.Fatal("the older attempt reopened the delivery") + } + requireSameTables(t, f, python) + }) +} + +func TestDEL25_receipt_recovery_keeps_the_dispatch_turn_and_its_provenance(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del25") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"in_progress"} + record := f.mustAttempt(event, nil) + turn := f.host.startTurn(parent, "", "inProgress", "") + request := str(record, "requestId") + f.host.ledger[request] = Obj{{Key: "requestId", Value: request}, {Key: "status", Value: "accepted"}, {Key: "resumed", Value: Obj{{Key: "approvalPolicy", Value: "never"}}}, {Key: "turnId", Value: turn.TurnID}} + outcome, err := NewReconciler(f.delivery).ReconcileAttempt(f.ctx, request, f.host, nil) + mustDo(t, err) + requireSameJSON(t, "record", record, python.Out["record"]) + requireSameJSON(t, "reconciled", outcome, python.Out["reconciled"]) + row := f.row(event) + if row.S("state") != Dispatched || row.S("dispatch_turn_id") != turn.TurnID || row.S("dispatch_evidence") != "transport_accepted" { + t.Fatalf("row %v", row) + } + requireSameTables(t, f, python) +} + +func TestDEL26_a_later_turn_needs_an_explicit_continuation_admission(t *testing.T) { + for _, tc := range []struct{ mode, reason, detail string }{ + {"none", "unassigned_turn", "explicit continuation admission"}, + {"valid", "", ""}, + {"anchor", "unassigned_turn", "is anchored to"}, + {"thread", "unassigned_turn", ""}, + {"malformed", "malformed_receipt", ""}, + {"replay", "", ""}, + } { + t.Run(tc.mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del26", tc.mode) + f := newFixture(t, tree) + rid := f.register(regOpts{}) + turn := turnRef{child, "turn-loop-3", "completed"} + text := "finished after several turns" + if tc.mode == "thread" { + turn.thread, text = "someone-else", "payload" + } + payload := f.readyPayload(rid, 1, []string{f.artifact("out.txt", text)}, 1, turn) + admission := []byte(`{"anchorTurnId": "turn-dispatch-1", "actor": "a", "reason": "b"}`) + switch tc.mode { + case "valid": + admission = []byte(`{"anchorTurnId": "turn-dispatch-1", "actor": "child-loop", "reason": "PABCD cycle 3 completed this generation"}`) + case "anchor": + admission = []byte(`{"anchorTurnId": "some-other-execution", "actor": "a", "reason": "b"}`) + case "malformed": + admission = []byte(`{"anchorTurnId": "turn-dispatch-1"}`) + case "none": + admission = nil + } + stored, err := f.accept(payload, store.AcceptOptions{Continuation: admission}) + if tc.mode == "replay" { + mustDo(t, err) + stored, err = f.accept(payload, store.AcceptOptions{}) + } + want := python.Out["result"].(map[string]any) + if tc.reason != "" { + requireReason(t, err, tc.reason) + requireSameJSON(t, "refusal", refusalOf(err), want) + if !strings.Contains(Detail(err), tc.detail) { + t.Fatalf("detail %q", Detail(err)) + } + } else { + mustDo(t, err) + ok := want["ok"].(map[string]any) + if tc.mode == "replay" && (!stored.Duplicate || ok["_duplicate"] != true) { + t.Fatal("a replay is a duplicate") + } + requireSameJSON(t, "stored record", loadsObj(stored.Record), withoutUnderscored(ok)) + } + requireSameTables(t, f, python) + }) + } +} + +func withoutUnderscored(m map[string]any) map[string]any { + out := map[string]any{} + for k, v := range m { + if !strings.HasPrefix(k, "_") { + out[k] = v + } + } + return out +} + +func TestDEL27_the_completion_message_is_a_verification_request_with_the_ack_instruction(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del27", "completion") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + message, err := f.delivery.PreviewMessage(f.ctx, event) + mustDo(t, err) + if message != python.Out["message"] { + t.Fatalf("message differs from Python:\n%s\n--\n%v", message, python.Out["message"]) + } + receipt, _ := f.delivery.Receipt(f.ctx, event) + manifest, _ := get(receipt, "manifest") + entry := manifest.([]any)[0].(Obj) + for _, want := range []string{"verification request", str(receipt, "revisionHash"), str(entry, "path"), str(entry, "sha256"), "ack-proof", "show --event"} { + if !strings.Contains(message, want) { + t.Fatalf("message lacks %q", want) + } + } + if strings.Contains(message, "None") { + t.Fatal("None in the message") + } +} + +// revisionFixture is DirectionalMessages._revision: acknowledged, ruled needs_changes. +func revisionFixture(t *testing.T, tree string) (*fixture, *Ack, string, string, Obj, Obj) { + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{recipients: []string{parent, child}}) + f.mustAttempt(event, nil) + f.clock.Advance(5) + turn := f.host.startTurn(parent, "ack-turn", "inProgress", "") + ack := NewAck(f.delivery) + acked, err := ack.Acknowledge(f.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, f.host) + mustDo(t, err) + verdict, err := ack.RecordVerdict(f.ctx, event, "needs_changes", "verdict-1", []any{Obj{{Key: "id", Value: "c-1"}, {Key: "verdict", Value: "needs_changes"}, {Key: "note", Value: "the manifest omits the migration script"}}}, nil, nil, nil) + mustDo(t, err) + revision := f.one("SELECT * FROM deliveries WHERE kind = 'revision_request'").S("event_id") + return f, ack, event, revision, acked, verdict +} + +func TestDEL28_the_revision_message_asks_for_no_acknowledgement_and_says_what_to_change(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del27", "revision") + f, _, source, revision, acked, verdict := revisionFixture(t, tree) + requireSameJSON(t, "ack", acked, python.Out["ack"]) + requireSameJSON(t, "verdict", verdict, python.Out["verdict"]) + message, err := f.delivery.PreviewMessage(f.ctx, revision) + mustDo(t, err) + if message != python.Out["message"] { + t.Fatalf("message differs from Python:\n%s\n--\n%v", message, python.Out["message"]) + } + for _, want := range []string{"revision request", "nothing to acknowledge", "emit --relationship", "the manifest omits the migration script", source, "--generation 2"} { + if !strings.Contains(message, want) { + t.Fatalf("message lacks %q", want) + } + } + for _, never := range []string{"ack-proof --event", "--ack-proof", "--ack-turn", "None"} { + if strings.Contains(message, never) { + t.Fatalf("message carries %q", never) + } + } +} + +func TestDEL29_the_instruction_each_side_is_given_is_the_one_that_works(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del27", "revision") + f, ack, source, revision, _, _ := revisionFixture(t, tree) + if f.one("SELECT 1 AS x FROM acks WHERE event_id = ?", source) == nil { + t.Fatal("the parent's acknowledgement landed") + } + _, err := ack.Acknowledge(f.ctx, revision, "child-turn", AckProof(revision, "child-turn"), true, nil, f.host) + requireReason(t, err, WrongDeliveryKind) + requireSameJSON(t, "child ack refusal", refusalOf(err), python.Out["childAck"]) + r, err := LoadRelationship(f.ctx, f.store, f.rid) + mustDo(t, err) + if r.Generation != 2 || python.Out["generation"] != float64(2) { + t.Fatal("the child's emit goes to generation 2") + } + requireSameTables(t, f, python) +} + +func TestDEL30_project_key_distinguishes_projects_for_a_shared_service(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del30") + f := newFixture(t, tree) + other := f.otherAssignment() + mine := f.register(regOpts{issue: "REL-3", dispatchRequest: "dispatch-3"}) + a, err := LoadRelationship(f.ctx, f.store, mine) + mustDo(t, err) + b, err := LoadRelationship(f.ctx, f.store, other) + mustDo(t, err) + if ProjectKey(a) != python.Out["mine"] || ProjectKey(b) != python.Out["other"] || ProjectKey(b) != "/other" || ProjectKey(a) == ProjectKey(b) { + t.Fatalf("project keys %q %q", ProjectKey(a), ProjectKey(b)) + } +} diff --git a/internal/relay/delivery/delivery_d_test.go b/internal/relay/delivery/delivery_d_test.go new file mode 100644 index 00000000..b99f315d --- /dev/null +++ b/internal/relay/delivery/delivery_d_test.go @@ -0,0 +1,181 @@ +package delivery + +import ( + "math" + "path/filepath" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_delivery.py properties DEL-31..DEL-35. + +func TestDEL31_a_restart_keeps_every_durable_record_and_resends_nothing(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del31") + f := newFixture(t, tree) + first := f.queuedEvent(regOpts{}) + firstRecord := f.mustAttempt(first, nil) + payload := f.readyPayload(f.rid, 1, []string{f.artifact("second.txt", "still in flight")}, 2, assigned("completed")) + _, err := f.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + second := str(payload, "eventId") + _, err = f.delivery.Enqueue(f.ctx, second, "", "") + mustDo(t, err) + f.host.script = []string{"transport_unknown"} + later := f.clock.Now() + 3600 + secondRecord := f.mustAttempt(second, at(later)) + requireSameJSON(t, "first", firstRecord, python.Out["first"]) + requireSameJSON(t, "second", secondRecord, python.Out["second"]) + _, err = execSQL(f.ctx, f.store, "INSERT INTO sync_targets (relationship_id, target, target_ref, recorded_at) VALUES (?,?,?,?) ON CONFLICT(relationship_id, target) DO UPDATE SET target_ref = excluded.target_ref, recorded_at = excluded.recorded_at", f.rid, "coordination_document", "DOC-1", f.clock.ISO()) + mustDo(t, err) + before := f.tables() + sends := len(f.host.sends) + + mustDo(t, f.store.Close()) + reopened, err := store.Open(f.ctx, filepath.Join(tree, "gostate", "relay.sqlite3"), "") + mustDo(t, err) + t.Cleanup(func() { _ = reopened.Close() }) + f.store = reopened + f.delivery = NewService(reopened, f.clock) + requireSameJSON(t, "durable records across the restart", f.tables(), before) + + recovered, err := NewReconciler(f.delivery).RecoverOnStart(f.ctx, f.host, nil) + mustDo(t, err) + requireSameJSON(t, "recovered", recovered, python.Out["recovered"]) + if len(f.host.sends) != sends { + t.Fatal("recovery sent") + } + awaiting, _ := get(recovered, "awaitingAck") + if len(awaiting.([]any)) != 1 || awaiting.([]any)[0] != first { + t.Fatalf("awaitingAck %v", awaiting) + } + attempts := f.row(first).I("attempt_count") + if again := f.mustAttempt(first, at(later+3600)); again != nil || f.row(first).I("attempt_count") != attempts || len(f.host.sends) != sends { + t.Fatal("a dispatched event replays nothing") + } + requireSameTables(t, f, python) +} + +func TestDEL32_a_stray_declaration_on_a_completion_is_not_labelled(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del32") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + receipt, err := f.delivery.Receipt(f.ctx, event) + mustDo(t, err) + receipt = set(receipt, "criteria", []any{Obj{{Key: "id", Value: "c1"}, {Key: "verdict", Value: "verified"}, {Key: "restoration", Value: "false"}}, Obj{{Key: "id", Value: "c2"}, {Key: "verdict", Value: "verified"}, {Key: "restoration", Value: true}}}) + _, err = execSQL(f.ctx, f.store, "UPDATE events SET receipt = ? WHERE event_id = ?", dumps(receipt), event) + mustDo(t, err) + message, err := f.delivery.PreviewMessage(f.ctx, event) + mustDo(t, err) + if message != python.Out["message"] || !strings.Contains(message, " c1: verified") || strings.Contains(message, "[restoration block]") { + t.Fatalf("message:\n%s", message) + } +} + +func TestDEL33_a_claim_refused_on_the_shared_gap_is_rescheduled_not_failed(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del33") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + now := f.clock.Now() + _, err := execSQL(f.ctx, f.store, "INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,1,?)", parent, math.Floor(now/3600)*3600, now) + mustDo(t, err) + f.delivery.RateLimited = func(string, float64) bool { return false } + if first := f.mustAttempt(event, at(now+1)); first != nil { + t.Fatal("paced") + } + row := f.row(event) + requireSameJSON(t, "row", row, python.Out["row"]) + if row.S("state") != Queued || !row.N("hold_reason") || row.I("attempt_count") != 0 || row.F("next_eligible_at") != now+1+f.delivery.Policy.MinSendInterval || f.count("SELECT COUNT(*) AS c FROM failed_operations") != 0 || len(f.host.sends) != 0 { + t.Fatalf("rolled back and rescheduled by the gap: %v", row) + } + f.delivery.RateLimited = nil + record := f.mustAttempt(event, at(row.F("next_eligible_at"))) + requireSameJSON(t, "record", record, python.Out["record"]) + requireSameTables(t, f, python) +} + +func TestDEL34_exec_source_recipients_deliver_or_withhold_with_their_relationship(t *testing.T) { + // The exec-aware archive check itself (bridge_adapter.is_archived over thread/list) is the + // bridge adapter's, todo 28; here the host answers the archive question it would answer, + // and discovery_cursors, which only that adapter writes, is left out of the comparison. + for _, mode := range []string{"live", "archived", "legacy"} { + t.Run(mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del34", mode) + f := newFixture(t, tree) + _, correction := f.correctionAfterNeedsChanges() + archived := mode != "live" + f.host.onArchived = func(thread string) (*bool, error) { + if thread == child { + return boolp(archived), nil + } + return boolp(false), nil + } + if mode == "legacy" { + _, err := execSQL(f.ctx, f.store, "INSERT INTO failed_operations (scope_key, operation, detail, error_code, occurred_at) VALUES (?, 'lifecycle_read', 'lifecycle_unknown', 'lifecycle_unknown', ?)", correction, f.clock.ISO()) + mustDo(t, err) + f.clock.Advance(1) + } + record := f.mustAttempt(correction, nil) + requireSameJSON(t, "record", record, python.Out["record"]) + failure := f.one("SELECT * FROM failed_operations WHERE scope_key = ? AND operation = 'lifecycle_read'", correction) + if mode == "live" { + if str(record, "deliveryState") != Dispatched || failure != nil || f.one("SELECT archived FROM recipient_lifecycle WHERE task_id = ?", child).I("archived") != 0 { + t.Fatal("a live exec child is delivered") + } + } else if f.row(correction).S("state") != WithheldPreSend || failure.S("error_code") != RecipientArchived || failure.S("relationship_id") != f.rid || failure.S("parent_task_id") != parent { + t.Fatalf("withheld with its relationship recorded: %v", failure) + } + requireSameTablesExcept(t, f, python, "discovery_cursors") + }) + } +} + +func TestDEL35_a_withhold_records_its_failure_in_its_own_transition(t *testing.T) { + for _, mode := range []string{"lifecycle", "settings", "busy"} { + t.Run(mode, func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del35", mode) + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{noSettings: mode == "settings"}) + switch mode { + case "lifecycle": + f.host.readFailures["is_archived"] = true + case "busy": + f.host.threads[parent].status = "active" + } + f.host.onGoalRead = func(string) { + _, err := execSQL(f.ctx, f.store, "UPDATE deliveries SET state = 'sending', attempt_count = attempt_count + 1 WHERE event_id = ?", event) + mustDo(t, err) + } + if record := f.mustAttempt(event, nil); record != nil { + t.Fatal("None") + } + operation := map[string]string{"lifecycle": "lifecycle_read", "settings": "settings_check", "busy": "parent_busy"}[mode] + if f.row(event).S("state") != Sending || f.one("SELECT * FROM failed_operations WHERE scope_key = ? AND operation = ?", event, operation) != nil { + t.Fatal("an overtaken withhold records nothing") + } + requireSameTables(t, f, python) + }) + } + t.Run("one stamp", func(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "del35", "stamp") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + f.host.threads[parent].archived = boolp(true) + f.clock.OnISO = func() { f.clock.T += 0.000001 } + if record := f.mustAttempt(event, nil); record != nil { + t.Fatal("None") + } + f.clock.OnISO = nil + if f.one("SELECT occurred_at FROM failed_operations WHERE scope_key = ? AND operation = 'lifecycle_read'", event).S("occurred_at") != f.row(event).S("updated_at") { + t.Fatal("one stamp") + } + requireSameTables(t, f, python) + }) +} diff --git a/internal/relay/delivery/delivery_relation_test.go b/internal/relay/delivery/delivery_relation_test.go new file mode 100644 index 00000000..961144ec --- /dev/null +++ b/internal/relay/delivery/delivery_relation_test.go @@ -0,0 +1,104 @@ +package delivery + +import ( + "context" + "os" + "testing" +) + +// test_delivery_relation.py DRL-1..DRL-4. DRL-0 (a drift guard between two Python spellings of +// the scope constants) is python-internal: Go has one constant. + +type stubReader struct { + answer Obj + asked []any +} + +func (r *stubReader) Up(_ context.Context, rid string) (Obj, error) { + r.asked = append(r.asked, Obj{{Key: "relationship_id", Value: rid}}) + return r.answer, nil +} + +func relationCases(t *testing.T) Obj { + raw, err := os.ReadFile("testdata/drl_answers.json") + mustDo(t, err) + v, err := loads(string(raw)) + mustDo(t, err) + return v.(Obj) +} + +func relationFixture() Relationship { + return Relationship{ID: "rel-1", Parent: Endpoint{TaskID: "parent-task"}, Child: Endpoint{TaskID: "child-task"}} +} + +func resolveCase(t *testing.T, f *fixture, cases Obj, name string) map[string]any { + c, _ := get(cases, name) + answer, _ := get(c.(Obj), "answer") + reader := &stubReader{answer: answer.(Obj)} + service := NewService(f.store, f.clock) + service.Linkage = reader + who, how, err := service.ResolveRecipient(f.ctx, relationFixture(), str(c.(Obj), "kind")) + var got map[string]any + if err != nil { + got = refusalOf(err) + } else { + got = map[string]any{"ok": []any{who, how}} + } + got["asked"] = reader.asked + return got +} + +func TestDRL01_an_unwired_service_uses_the_relationship_row(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "drl") + f := newFixture(t, tree) + for _, kind := range []string{Completion, Revision} { + who, how, err := f.delivery.ResolveRecipient(f.ctx, relationFixture(), kind) + mustDo(t, err) + name := map[string]string{Completion: "unwired_completion", Revision: "unwired_revision"}[kind] + requireSameJSON(t, name, []any{who, how}, python.Out[name]) + } +} + +func TestDRL02_an_agreeing_owner_resolves_verified_at_the_right_level(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "drl") + f := newFixture(t, tree) + cases := relationCases(t) + for _, name := range []string{"agree_completion", "agree_revision"} { + got := resolveCase(t, f, cases, name) + requireSameJSON(t, name, got, python.Out[name]) + if got["ok"] == nil { + t.Fatalf("%s refused: %v", name, got) + } + } +} + +func TestDRL03_linkage_refusals_keep_distinct_reasons_and_never_fall_back(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "drl") + f := newFixture(t, tree) + cases := relationCases(t) + for name, reason := range map[string]string{ + "owner_changed": RelationOwnerDrift, "drift_resolved": RelationOwnerDrift, "live_beside_audit": RelationOwnerDrift, + "unreadable": RelationUnreadable, "nothing_found": UnregisteredScope, "unregistered": UnregisteredScope, + "two_owners": DuplicateScopeOwner, "instruction_conflict": LinkConflict, "undefined_direction": NotClaimable, + } { + got := resolveCase(t, f, cases, name) + requireSameJSON(t, name, got, python.Out[name]) + if got["reason"] != reason { + t.Fatalf("%s: %v", name, got) + } + } +} + +func TestDRL04_a_retained_audit_conflict_does_not_block_a_healthy_delivery(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "drl") + f := newFixture(t, tree) + got := resolveCase(t, f, relationCases(t), "audit_only") + requireSameJSON(t, "audit_only", got, python.Out["audit_only"]) + if got["ok"] == nil { + t.Fatal("refused on an audit row") + } +} diff --git a/internal/relay/delivery/errors.go b/internal/relay/delivery/errors.go new file mode 100644 index 00000000..548ef111 --- /dev/null +++ b/internal/relay/delivery/errors.go @@ -0,0 +1,50 @@ +package delivery + +import ( + "errors" + "fmt" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Refusal reasons this package raises (errors.RefusalReason values, spelled as Python spells them). +const ( + NotClaimable = "not_claimable" + RecipientNotAuthorized = "recipient_not_authorized" + ScopeEscape = "scope_escape" + RelationshipNotActive = "relationship_not_active" + UnregisteredRelationship = "unregistered_relationship" + UnknownGeneration = "unknown_generation" + RelationUnreadable = "relation_unreadable" + RelationOwnerDrift = "relation_owner_drift" + DuplicateScopeOwner = "duplicate_scope_owner" + LinkConflict = "link_conflict" + UnregisteredScope = "unregistered_scope" + SettingsUnavailable = "settings_unavailable" + SettingsIncomplete = "settings_incomplete" + SettingsMistyped = "settings_mistyped" + UnsupportedSandboxType = "unsupported_sandbox_type" + UnsupportedApprovalPolicy = "unsupported_approval_policy" + RolePolicyUnconfigured = "role_policy_unconfigured" + RoleBindingMismatch = "role_binding_mismatch" +) + +// Refused is a RelayError: a machine reason and the human detail beside it. It reuses the store's +// refusal type, so a refusal from either package reads the same way. +type Refused = store.RefusedError + +func refuse(reason, format string, args ...any) error { + return &store.RefusedError{Reason: reason, Detail: fmt.Sprintf(format, args...)} +} + +// Reason is the refusal reason carried by err, or "". +func Reason(err error) string { return store.RefusalReason(err) } + +// Detail is the refusal detail carried by err. +func Detail(err error) string { + var r *store.RefusedError + if errors.As(err, &r) { + return r.Detail + } + return "" +} diff --git a/internal/relay/delivery/fakehost_test.go b/internal/relay/delivery/fakehost_test.go new file mode 100644 index 00000000..6f7230f0 --- /dev/null +++ b/internal/relay/delivery/fakehost_test.go @@ -0,0 +1,299 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "sync" +) + +// fakeThread and fakeHost are fakehost.FakeHostAdapter: a deterministic host shaped like the +// real one where the delivery logic depends on it. The receipt is written BEFORE the calls run, +// and a settled request id is answered from its receipt, never sent again. +type fakeThread struct { + id string + status string + approvalPolicy string + archived *bool + goalStatus any + canAcceptInput bool + turns []TurnInfo + items [][3]string +} + +type fakeSend struct { + requestID, thread, message, outcome string +} + +type fakeHost struct { + clock *FakeClock + threads map[string]*fakeThread + ledger map[string]Obj + script []string + sends []fakeSend + readFailures map[string]bool + counter int + calls []string + onGoalRead func(thread string) + onArchived func(thread string) (*bool, error) + // scanLimit is FakeHostAdapter.scan_limit: a bound below the caller's on item scans. + scanLimit int + // mu serialises the call log: two parents acknowledging at one instant (MPI-2) read the + // host from two goroutines, which the Python fake never had to survive under its GIL. + mu sync.Mutex +} + +func newFakeHost(clock *FakeClock) *fakeHost { + return &fakeHost{clock: clock, threads: map[string]*fakeThread{}, ledger: map[string]Obj{}, readFailures: map[string]bool{}} +} + +func (h *fakeHost) addThread(id string) *fakeThread { + f := false + t := &fakeThread{id: id, status: "idle", approvalPolicy: "never", archived: &f, canAcceptInput: true} + h.threads[id] = t + return t +} + +func (h *fakeHost) startTurn(thread, turnID, status, text string) TurnInfo { + t := h.threads[thread] + h.counter++ + if turnID == "" { + turnID = fmt.Sprintf("turn-%s-%d", thread, h.counter) + } + at := h.clock.Now() + turn := TurnInfo{TurnID: turnID, Status: status, StartedAt: &at} + t.turns = append(t.turns, turn) + if text != "" { + t.items = append(t.items, [3]string{turnID, text, "userMessage"}) + } + return turn +} + +func (h *fakeHost) finishTurn(thread, turnID, status string) { + for i, turn := range h.threads[thread].turns { + if turn.TurnID == turnID { + h.threads[thread].turns[i].Status = status + } + } +} + +func (h *fakeHost) guard(name string) error { + h.mu.Lock() + defer h.mu.Unlock() + h.calls = append(h.calls, name) + if h.readFailures[name] { + return &HostError{Kind: "ConnectionError", Message: name + " unavailable"} + } + return nil +} + +func (h *fakeHost) ReadThread(thread string) (ThreadFacts, error) { + if err := h.guard("read_thread"); err != nil { + return ThreadFacts{}, err + } + t := h.threads[thread] + accepts := t.canAcceptInput + return ThreadFacts{RuntimeStatus: t.status, CanAcceptInput: &accepts}, nil +} + +func (h *fakeHost) IsArchived(thread string, _ any) (*bool, error) { + if err := h.guard("is_archived"); err != nil { + return nil, err + } + if h.onArchived != nil { + return h.onArchived(thread) + } + return h.threads[thread].archived, nil +} + +func (h *fakeHost) ReadGoalStatus(thread string) (any, error) { + if h.onGoalRead != nil { + h.onGoalRead(thread) + } + if err := h.guard("read_goal_status"); err != nil { + return nil, err + } + return h.threads[thread].goalStatus, nil +} + +func (h *fakeHost) ListTurnIDs(thread string, limit int) ([]string, error) { + if err := h.guard("list_turn_ids"); err != nil { + return nil, err + } + turns := h.threads[thread].turns + if len(turns) > limit { + turns = turns[len(turns)-limit:] + } + var ids []string + for _, t := range turns { + ids = append(ids, t.TurnID) + } + return ids, nil +} + +func (h *fakeHost) ReadTurn(thread, turn string) (*TurnInfo, error) { + if err := h.guard("read_turn"); err != nil { + return nil, err + } + for _, t := range h.threads[thread].turns { + if t.TurnID == turn { + copy := t + return ©, nil + } + } + return nil, nil +} + +func (h *fakeHost) GetOperation(requestID string) (Obj, error) { + if err := h.guard("get_operation"); err != nil { + return nil, err + } + return h.ledger[requestID], nil +} + +func (h *fakeHost) FindToken(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + if err := h.guard("find_token"); err != nil { + return TokenScan{}, err + } + items := h.threads[thread].items + bound := h.bound(limit) + scanned := 0 + for i := len(items) - 1; i >= 0 && scanned < bound; i-- { + scanned++ + item := items[i] + if messageOnly && !isMessage(item[2]) { + continue + } + if strings.Contains(item[1], token) { + return TokenScan{Found: true, TurnID: item[0], Exhausted: scanned >= len(items), Scanned: scanned}, nil + } + } + return TokenScan{Found: false, Exhausted: bound >= len(items), Scanned: scanned}, nil +} + +func (h *fakeHost) bound(limit int) int { + if h.scanLimit > 0 { + return min(limit, h.scanLimit) + } + return limit +} + +// RecipientFingerprint is FakeHostAdapter.recipient_fingerprint (window 8). +func (h *fakeHost) RecipientFingerprint(thread string) (string, error) { + if err := h.guard("recipient_fingerprint"); err != nil { + return "", err + } + digest := sha256.New() + for i, item := range h.newestItems(thread) { + if i == 8 { + break + } + body := sha256.Sum256([]byte(item.Text)) + digest.Write([]byte(item.Turn + ":" + hex.EncodeToString(body[:]) + "|")) + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +func (h *fakeHost) SendMessage(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + if cached, ok := h.ledger[requestID]; ok && str(cached, "status") != Unfinished { + return append(append(Obj(nil), cached...), F{Key: "replayed", Value: true}), nil + } + outcome := "accepted" + if len(h.script) > 0 { + outcome, h.script = h.script[0], h.script[1:] + } + receipt := Obj{{Key: "requestId", Value: requestID}, {Key: "operation", Value: "send_message_to_thread"}, {Key: "status", Value: Unfinished}, {Key: "threadId", Value: thread}, {Key: "retrySafe", Value: false}} + h.ledger[requestID] = receipt + h.sends = append(h.sends, fakeSend{requestID, thread, message, outcome}) + t := h.threads[thread] + resumed := Obj{{Key: "approvalPolicy", Value: t.approvalPolicy}} + rpc := func(code, msg string) Obj { return Obj{{Key: "code", Value: code}, {Key: "message", Value: msg}} } + switch outcome { + case "in_progress": + return append(Obj(nil), receipt...), nil + case "process_death": + return nil, &HostError{Kind: "ProcessDied", Message: "the relay process was killed mid-send"} + case "busy": + receipt = set(receipt, "status", FailedStatus) + receipt = set(receipt, "error", "thread/read: Thread is active; message withheld. Wait for completion.") + receipt = set(receipt, "rpcError", rpc("thread_busy", "Thread is active")) + case "read_fail": + receipt = set(receipt, "status", FailedStatus) + receipt = set(receipt, "error", "thread/read: transport refused") + receipt = set(receipt, "rpcError", rpc("internal", "transport refused")) + case "approval_policy": + receipt = set(receipt, "status", FailedStatus) + receipt = set(receipt, "resumed", resumed) + receipt = set(receipt, "error", "thread/resume: Interactive approvals unsupported; message withheld.") + receipt = set(receipt, "rpcError", rpc("unsupported_approval_policy", "unsupported")) + case "turn_start_fail": + receipt = set(receipt, "status", FailedStatus) + receipt = set(receipt, "resumed", resumed) + receipt = set(receipt, "error", "turn/start: refused") + receipt = set(receipt, "rpcError", rpc("internal", "refused")) + case "transport_unknown": + receipt = set(receipt, "status", OutcomeUnknown) + receipt = set(receipt, "error", "TransportError: turn/start: response unavailable; do not resend") + case "steer_existing": + var existing any + if n := len(t.turns); n > 0 { + existing = t.turns[n-1].TurnID + } + receipt = set(receipt, "status", Accepted) + receipt = set(receipt, "resumed", resumed) + receipt = set(receipt, "turnId", existing) + t.items = append(t.items, [3]string{pyStr(existing), message, "userMessage"}) + default: + turn := h.startTurn(thread, "", "inProgress", message) + receipt = set(receipt, "status", Accepted) + receipt = set(receipt, "resumed", resumed) + receipt = set(receipt, "turnId", turn.TurnID) + } + h.ledger[requestID] = receipt + return append(Obj(nil), receipt...), nil +} + +func (h *fakeHost) newestItems(thread string) []Item { + items := h.threads[thread].items + out := make([]Item, 0, len(items)) + for i := len(items) - 1; i >= 0; i-- { + out = append(out, Item{items[i][0], items[i][1], items[i][2]}) + } + return out +} + +func (h *fakeHost) FindDispatchedTurn(thread, turnID string, sentAt float64) (TurnPresence, error) { + if err := h.guard("find_dispatched_turn"); err != nil { + return TurnPresence{}, err + } + turns := h.threads[thread].turns + newest := make([]TurnInfo, 0, len(turns)) + for i := len(turns) - 1; i >= 0; i-- { + newest = append(newest, turns[i]) + } + return FindInListing([]ListingPage{{newest, false}}, turnID, sentAt) +} + +func (h *fakeHost) FindTokenSince(thread, token string, older []string, limit int) (TokenScan, error) { + if err := h.guard("find_token_since"); err != nil { + return TokenScan{}, err + } + items := h.newestItems(thread) + bound := h.bound(limit) + return FindTokenIn([]ItemPage{{items[:min(bound, len(items))], bound < len(items)}}, token, older), nil +} + +func (h *fakeHost) FindTokenInTurn(thread, token, turnID string, limit int) (TokenScan, error) { + if err := h.guard("find_token_in_turn"); err != nil { + return TokenScan{}, err + } + var own []Item + for _, item := range h.threads[thread].items { + if item[0] == turnID { + own = append(own, Item{item[0], item[1], item[2]}) + } + } + bound := min(limit, len(own)) + return FindTokenInTurnItems([]ItemPage{{own[:bound], limit < len(own)}}, token, turnID), nil +} diff --git a/internal/relay/delivery/harness_test.go b/internal/relay/delivery/harness_test.go new file mode 100644 index 00000000..7163d4c1 --- /dev/null +++ b/internal/relay/delivery/harness_test.go @@ -0,0 +1,530 @@ +package delivery + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "slices" + "sort" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Fixture identities match Python's tests/support.py. +const ( + parent = "01parent-task" + child = "01child-task" + issue = "REL-1" + host = "host-a" + dispatchTurn = "turn-dispatch-1" +) + +func repoRoot(t *testing.T) string { + t.Helper() + _, file, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(file), "..", "..", "..") +} + +// pyRun is one Python scenario over tree: its out block, every non-empty table, and its sends. +type pyRun struct { + Out map[string]any `json:"out"` + Tables map[string][]map[string]any `json:"tables"` + Sends [][]any `json:"sends"` +} + +// runPython drives testdata/scenarios/.py through the real Python package over tree. +func runPython(t *testing.T, tree, name string, args ...string) pyRun { + t.Helper() + root := repoRoot(t) + script, _ := filepath.Abs("testdata/pyscenario.py") + cmd := exec.Command("uv", append([]string{"run", "--no-sync", "python", script, tree, name}, args...)...) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + home := t.TempDir() + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "state"), "XDG_DATA_HOME="+filepath.Join(home, "data"), "XDG_CONFIG_HOME="+filepath.Join(home, "config"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, "PYTHONPATH="+filepath.Join(root, "packages", "codex-session-relay", "src")+":"+filepath.Join(root, "packages", "codex-session-relay")) + output, err := cmd.Output() + if err != nil { + stderr := "" + if e, ok := err.(*exec.ExitError); ok { + stderr = string(e.Stderr) + } + t.Fatalf("python scenario %s: %v\n%s", name, err, stderr) + } + lines := strings.Split(strings.TrimSpace(string(output)), "\n") + var run pyRun + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &run); err != nil { + t.Fatalf("python scenario %s output: %v\n%s", name, err, output) + } + return run +} + +// fixture is DeliveryTestCase: a registered relationship, a final event, a fake host. +type fixture struct { + t *testing.T + ctx context.Context + tree string + root string + clock *FakeClock + store *store.Store + intake store.ReceiptIntake + delivery *Service + host *fakeHost + rid string + // skipTables are left out of a table comparison (tableFilter). + skipTables []string +} + +// newFixture shares tree with a Python run when tree is non-empty; the Go store lives beside it. +func newFixture(t *testing.T, tree string) *fixture { + t.Helper() + if tree == "" { + tree = t.TempDir() + } + root := filepath.Join(tree, "work") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("XDG_STATE_HOME", filepath.Join(tree, "xdg-state")) + ctx := context.Background() + s, err := store.Open(ctx, filepath.Join(tree, "gostate", "relay.sqlite3"), "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + clock := NewFakeClock() + f := &fixture{t: t, ctx: ctx, tree: tree, root: root, clock: clock, store: s, host: newFakeHost(clock)} + f.intake = store.ReceiptIntake{Store: s, Now: clock.ISO, Minimum: store.BestEffortDetection} + f.delivery = NewService(s, clock) + f.host.addThread(parent) + f.host.addThread(child) + return f +} + +func (f *fixture) artifact(name, text string) string { + path := filepath.Join(f.root, name) + if err := os.WriteFile(path, []byte(text), 0o644); err != nil { + f.t.Fatal(err) + } + return path +} + +func mustDo(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} + +func taskSettings(cwd string) string { return taskSettingsWith(cwd, "never") } + +func taskSettingsWith(cwd, approval string) string { + return dumpsSorted(Obj{ + {Key: "sandbox", Value: Obj{{Key: "type", Value: "workspaceWrite"}, {Key: "writableRoots", Value: []any{}}, {Key: "networkAccess", Value: false}, {Key: "excludeTmpdirEnvVar", Value: false}, {Key: "excludeSlashTmp", Value: false}}}, + {Key: "approvalPolicy", Value: approval}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}, + {Key: "model", Value: "anthropic/claude-opus-5"}, {Key: "reasoningEffort", Value: "xhigh"}, + {Key: "environments", Value: []any{Obj{{Key: "environmentId", Value: "local"}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}}}}, + }) +} + +type regOpts struct { + issue, dispatchRequest string + // other registers support's other_assignment endpoints (CrossAssignmentDelivery). + parent, parentCwd, parentSession, child, childRoot, childSession, turn string + recipients []string + noSettings bool + parentSettings string + scopeRef any + // parentOnlySettings records the parent's settings only (TwoParents.assignment). + parentOnlySettings bool +} + +// register writes what Registry.register and record_settings write for the fixture (the +// registration port is todo 25; these are the same rows, compared with Python's below). +func (f *fixture) register(o regOpts) string { + if o.issue == "" { + o.issue = issue + } + if o.dispatchRequest == "" { + o.dispatchRequest = "dispatch-1" + } + if o.parent == "" { + o.parent, o.parentCwd, o.parentSession, o.child, o.childRoot, o.childSession, o.turn = parent, "/parent", "cxc-parent", child, f.root, "cxc-child", dispatchTurn + } + if o.recipients == nil { + o.recipients = []string{o.parent} + } + rid, err := store.RelationshipID(o.parent, o.child, o.issue) + mustDo(f.t, err) + now := f.clock.ISO() + recipients := make([]any, len(o.recipients)) + for i, r := range o.recipients { + recipients[i] = r + } + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, f.store, "INSERT INTO relationships (relationship_id, issue_key, status, parent_task_id, parent_host_id, parent_cwd, parent_cxc_session, child_task_id, child_host_id, child_cwd, child_cxc_session, execution_generation, artifact_roots, allowed_recipients, scope_ref, supersedes, superseded_by, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL,?,?)", + rid, o.issue, "active", o.parent, host, o.parentCwd, nullable(o.parentSession), o.child, host, o.childRoot, nullable(o.childSession), 1, dumps([]any{o.childRoot}), dumps(recipients), o.scopeRef, nil, now, now); err != nil { + return err + } + if _, err := execSQL(ctx, f.store, "INSERT INTO generations (relationship_id, execution_generation, dispatch_request_id, anchor_state, dispatch_turn_id, reason, opened_at, bound_at) VALUES (?,?,?,?,?,?,?,?)", rid, 1, o.dispatchRequest, "bound", o.turn, "initial_assignment", now, now); err != nil { + return err + } + return journal(ctx, f.store, "relationship_registered", rid, Obj{{Key: "issueKey", Value: o.issue}}, now) + })) + switch { + case o.parentSettings != "": + _, err := execSQL(f.ctx, f.store, "INSERT INTO authorized_settings (task_id, settings, source, recorded_at) VALUES (?,?,?,?)", parent, o.parentSettings, "test-raw", now) + mustDo(f.t, err) + case o.parentOnlySettings: + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, f.store, "INSERT INTO authorized_settings (task_id, settings, source, recorded_at) VALUES (?,?,?,?)", o.parent, taskSettings(o.parentCwd), "creation_result", now); err != nil { + return err + } + return journal(ctx, f.store, "settings_recorded", o.parent, Obj{{Key: "source", Value: "creation_result"}}, now) + })) + case !o.noSettings && o.parent == parent: + for _, task := range []struct{ id, cwd string }{{parent, "/parent"}, {child, f.root}} { + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, f.store, "INSERT INTO authorized_settings (task_id, settings, source, recorded_at) VALUES (?,?,?,?) ON CONFLICT(task_id) DO UPDATE SET settings = excluded.settings, source = excluded.source, recorded_at = excluded.recorded_at", task.id, taskSettings(task.cwd), "creation_result", now); err != nil { + return err + } + return journal(ctx, f.store, "settings_recorded", task.id, Obj{{Key: "source", Value: "creation_result"}}, now) + })) + } + } + if o.parent == parent { + f.rid = rid + } + return rid +} + +// otherAssignment is CrossAssignmentDelivery.other_assignment. +func (f *fixture) otherAssignment() string { + root := filepath.Join(f.tree, "other-project") + mustDo(f.t, os.MkdirAll(root, 0o755)) + return f.register(regOpts{issue: "REL-2", dispatchRequest: "dispatch-2", parent: "01other-parent", parentCwd: "/other", parentSession: "cxc-other", child: "01other-child", childRoot: root, childSession: "cxc-other-c", turn: "turn-dispatch-2", noSettings: true}) +} + +type turnRef struct{ thread, turn, status string } + +func assigned(status string) turnRef { return turnRef{child, dispatchTurn, status} } + +// readyPayload is support.ready_payload, in Python's key order. +func (f *fixture) readyPayload(rid string, generation int64, paths []string, attempt int, turn turnRef) Obj { + entries, err := store.BuildManifest(paths, []string{f.root}) + mustDo(f.t, err) + revision, err := store.ManifestRevision(entries) + mustDo(f.t, err) + event, err := store.EventID(rid, int(generation), revision, "ready_for_review", turn.turn, &attempt) + mustDo(f.t, err) + manifest := make([]any, len(entries)) + for i, e := range entries { + manifest[i] = Obj{{Key: "path", Value: e.Path}, {Key: "sha256", Value: e.SHA256}, {Key: "bytes", Value: *e.Bytes}} + } + return Obj{{Key: "eventId", Value: event}, {Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: generation}, {Key: "attempt", Value: int64(attempt)}, {Key: "revisionHash", Value: revision}, {Key: "outcome", Value: "ready_for_review"}, {Key: "producer", Value: "child"}, + {Key: "turnRef", Value: Obj{{Key: "threadId", Value: turn.thread}, {Key: "turnId", Value: turn.turn}, {Key: "turnStatus", Value: turn.status}}}, {Key: "manifest", Value: manifest}, {Key: "emittedAt", Value: f.clock.ISO()}} +} + +// executionPayload is support.execution_payload. +func (f *fixture) executionPayload(rid string, generation int64, outcome string, attempt int, turn turnRef) Obj { + event, err := store.EventID(rid, int(generation), store.NoDeliverable, outcome, turn.turn, &attempt) + mustDo(f.t, err) + return Obj{{Key: "eventId", Value: event}, {Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: generation}, {Key: "attempt", Value: int64(attempt)}, {Key: "revisionHash", Value: store.NoDeliverable}, {Key: "outcome", Value: outcome}, {Key: "producer", Value: "child"}, + {Key: "turnRef", Value: Obj{{Key: "threadId", Value: turn.thread}, {Key: "turnId", Value: turn.turn}, {Key: "turnStatus", Value: turn.status}}}, {Key: "manifest", Value: nil}, {Key: "emittedAt", Value: f.clock.ISO()}} +} + +func (f *fixture) accept(payload Obj, options store.AcceptOptions) (store.StoredReceipt, error) { + turn, _ := get(payload, "turnRef") + t := turn.(Obj) + return f.intake.AcceptChildReceiptWith(f.ctx, []byte(dumps(payload)), store.TurnReference{ThreadID: str(t, "threadId"), TurnID: str(t, "turnId"), Status: str(t, "turnStatus")}, options) +} + +// readyEvent is DeliveryTestCase.ready_event. +func (f *fixture) readyEvent(o regOpts) string { + rid := f.register(o) + payload := f.readyPayload(rid, 1, []string{f.artifact("out.txt", "the deliverable")}, 1, assigned("completed")) + _, err := f.accept(payload, store.AcceptOptions{}) + mustDo(f.t, err) + return str(payload, "eventId") +} + +// queuedEvent is DeliveryTestCase.queued_event. +func (f *fixture) queuedEvent(o regOpts) string { + event := f.readyEvent(o) + _, err := f.delivery.Enqueue(f.ctx, event, "", "") + mustDo(f.t, err) + return event +} + +func (f *fixture) attempt(event string, now *float64) (Obj, error) { + return f.delivery.Attempt(f.ctx, event, f.host, now, "") +} + +func (f *fixture) mustAttempt(event string, now *float64) Obj { + f.t.Helper() + record, err := f.attempt(event, now) + mustDo(f.t, err) + return record +} + +func (f *fixture) row(event string) Row { + f.t.Helper() + row, err := f.delivery.Get(f.ctx, event) + mustDo(f.t, err) + return row +} + +func (f *fixture) one(query string, args ...any) Row { + f.t.Helper() + row, err := one(f.ctx, f.store, query, args...) + mustDo(f.t, err) + return row +} + +func (f *fixture) count(query string, args ...any) int64 { + return f.one(query, args...).I("c") +} + +func (f *fixture) eligible() []string { + rows, err := f.delivery.Eligible(f.ctx, f.clock.Now(), 10, 0, 0, nil) + mustDo(f.t, err) + var ids []string + for _, r := range rows { + ids = append(ids, r.S("event_id")) + } + return ids +} + +func at(v float64) *float64 { return &v } + +// tables dumps every non-empty table of the Go store, as the Python harness dumps its own. +func (f *fixture) tables() map[string][]map[string]any { + names, err := all(f.ctx, f.store, "SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ('schema_meta','sqlite_sequence') ORDER BY name") + mustDo(f.t, err) + out := map[string][]map[string]any{} + for _, n := range names { + rows, err := all(f.ctx, f.store, "SELECT * FROM "+n.S("name")+" ORDER BY rowid") + mustDo(f.t, err) + if len(rows) == 0 || slices.Contains(f.skipTables, n.S("name")) { + continue + } + for _, r := range rows { + out[n.S("name")] = append(out[n.S("name")], map[string]any(r)) + } + } + return out +} + +func normalizeJSON(t *testing.T, v any) any { + raw, err := json.Marshal(v) + mustDo(t, err) + var out any + mustDo(t, json.Unmarshal(raw, &out)) + return out +} + +// requireSameTables compares the Go store with the Python one, every row of every table. +func requireSameTables(t *testing.T, f *fixture, python pyRun) { + t.Helper() + got := normalizeJSON(t, f.tables()).(map[string]any) + want := normalizeJSON(t, python.Tables).(map[string]any) + var names []string + for n := range want { + names = append(names, n) + } + for n := range got { + if _, ok := want[n]; !ok { + names = append(names, n) + } + } + sort.Strings(names) + for _, n := range names { + if !reflect.DeepEqual(got[n], want[n]) { + g, _ := json.MarshalIndent(got[n], "", " ") + w, _ := json.MarshalIndent(want[n], "", " ") + t.Errorf("table %s differs from Python\ngo: %s\npython: %s", n, g, w) + } + } +} + +// requireSameJSON compares one Go value with the Python value of the same name. +func requireSameJSON(t *testing.T, what string, got any, want any) { + t.Helper() + g, w := normalizeJSON(t, jsonable(got)), normalizeJSON(t, want) + if !reflect.DeepEqual(g, w) { + gb, _ := json.Marshal(g) + wb, _ := json.Marshal(w) + t.Errorf("%s differs from Python\ngo: %s\npython: %s", what, gb, wb) + } +} + +// jsonable turns an Obj tree into maps for comparison. +func jsonable(v any) any { + switch t := v.(type) { + case Obj: + if t == nil { + return nil + } + m := map[string]any{} + for _, f := range t { + m[f.Key] = jsonable(f.Value) + } + return m + case []any: + out := make([]any, len(t)) + for i, x := range t { + out[i] = jsonable(x) + } + return out + case Row: + return map[string]any(t) + case map[string]any: + m := map[string]any{} + for k, x := range t { + m[k] = jsonable(x) + } + return m + case error: + return map[string]any{"reason": Reason(t), "detail": Detail(t)} + } + return v +} + +func refusalOf(err error) map[string]any { + return map[string]any{"reason": Reason(err), "detail": Detail(err)} +} + +func requireReason(t *testing.T, err error, reason string) { + t.Helper() + if Reason(err) != reason { + t.Fatalf("want refusal %s, got %v", reason, err) + } +} + +func sendsJSON(h *fakeHost) []any { + out := []any{} + for _, s := range h.sends { + out = append(out, []any{s.requestID, s.thread, s.message, s.outcome}) + } + return out +} + +// setStatus is registry.set_status for an unscoped relationship: the same rows Python writes. +func (f *fixture) setStatus(status string) { f.setStatusBy(status, "user") } + +func (f *fixture) setStatusBy(status, actor string) { + now := f.clock.ISO() + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, f.store, "UPDATE relationships SET status = ?, updated_at = ? WHERE relationship_id = ?", status, now, f.rid); err != nil { + return err + } + return journal(ctx, f.store, "status_changed", f.rid, Obj{{Key: "status", Value: status}, {Key: "actor", Value: actor}}, now) + })) +} + +// resume is registry.resume restating the current scope. +func (f *fixture) resume() { + f.setStatus("active") +} + +// supersede is registry.supersede. +func (f *fixture) supersede(successor string) { + now := f.clock.ISO() + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, f.store, "UPDATE relationships SET superseded_by = ?, status = 'archived', updated_at = ? WHERE relationship_id = ?", successor, now, f.rid); err != nil { + return err + } + return journal(ctx, f.store, "superseded", f.rid, Obj{{Key: "by", Value: successor}}, now) + })) +} + +func boolp(b bool) *bool { return &b } + +// counted wraps the fake host and counts every pre-claim read (CountedHostReads). +type counted struct { + *fakeHost + calls []string +} + +func (c *counted) ReadThread(t string) (ThreadFacts, error) { + c.calls = append(c.calls, "read_thread") + return c.fakeHost.ReadThread(t) +} +func (c *counted) IsArchived(t string, cwd any) (*bool, error) { + c.calls = append(c.calls, "is_archived") + return c.fakeHost.IsArchived(t, cwd) +} +func (c *counted) ReadGoalStatus(t string) (any, error) { + c.calls = append(c.calls, "read_goal_status") + return c.fakeHost.ReadGoalStatus(t) +} +func (c *counted) ListTurnIDs(t string, limit int) ([]string, error) { + c.calls = append(c.calls, "list_turn_ids") + return c.fakeHost.ListTurnIDs(t, limit) +} + +// correctionAfterNeedsChanges is support.correction_after_needs_changes. +func (f *fixture) correctionAfterNeedsChanges() (string, string) { + event := f.queuedEvent(regOpts{recipients: []string{parent, child}}) + f.mustAttempt(event, nil) + f.clock.Advance(5) + turn := f.host.startTurn(parent, "ack-turn", "inProgress", "") + ack := NewAck(f.delivery) + _, err := ack.Acknowledge(f.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, f.host) + mustDo(f.t, err) + _, err = ack.RecordVerdict(f.ctx, event, "needs_changes", "v1", nil, []any{Obj{{Key: "id", Value: "c1"}, {Key: "verdict", Value: "needs_changes"}, {Key: "note", Value: "fix the shape"}}}, nil, nil) + mustDo(f.t, err) + correction := f.one("SELECT event_id FROM deliveries WHERE relationship_id = ? AND kind = ?", f.rid, Revision).S("event_id") + f.clock.Advance(1) + return event, correction +} + +// requireSameTablesExcept compares every table but the named ones. +func requireSameTablesExcept(t *testing.T, f *fixture, python pyRun, skip ...string) { + t.Helper() + for _, name := range skip { + delete(python.Tables, name) + } + g := &tableFilter{f, skip} + requireSameTables(t, g.fixture(), python) +} + +type tableFilter struct { + f *fixture + skip []string +} + +func (tf *tableFilter) fixture() *fixture { + c := *tf.f + c.skipTables = tf.skip + return &c +} + +// pythonValue runs one line of Python against the package and returns its stdout, trimmed. +func pythonValue(t *testing.T, script string) string { + t.Helper() + root := repoRoot(t) + cmd := exec.Command("uv", "run", "--no-sync", "python", "-c", script) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + home := t.TempDir() + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "state"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home) + out, err := cmd.Output() + mustDo(t, err) + return strings.TrimSpace(string(out)) +} + +// rawSettings is json.dumps(task_settings(cwd, approvalPolicy=...)) in its insertion order, as +// support.register writes a raw settings row. +func rawSettings(cwd, approval string) string { + return dumps(Obj{ + {Key: "sandbox", Value: Obj{{Key: "type", Value: "workspaceWrite"}, {Key: "writableRoots", Value: []any{}}, {Key: "networkAccess", Value: false}, {Key: "excludeTmpdirEnvVar", Value: false}, {Key: "excludeSlashTmp", Value: false}}}, + {Key: "approvalPolicy", Value: approval}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}, + {Key: "model", Value: "anthropic/claude-opus-5"}, {Key: "reasoningEffort", Value: "xhigh"}, + {Key: "environments", Value: []any{Obj{{Key: "environmentId", Value: "local"}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}}}}, + }) +} diff --git a/internal/relay/delivery/hostadapter.go b/internal/relay/delivery/hostadapter.go new file mode 100644 index 00000000..849348f1 --- /dev/null +++ b/internal/relay/delivery/hostadapter.go @@ -0,0 +1,199 @@ +package delivery + +import ( + "fmt" + "slices" + "strings" +) + +// Host reading rules shared by every adapter (hostadapter.py). +const ( + TurnStartPrecisionSeconds = 1.0 + DispatchTurnSkewSeconds = 60.0 + TurnPresent = "present" + TurnAbsent = "absent" + InTurnItemsMax = 2000 + userMessage = "userMessage" +) + +var agentOutput = []string{"agentMessage", "collabAgentToolCall", "commandExecution", "contextCompaction", "dynamicToolCall", "enteredReviewMode", "exitedReviewMode", "fileChange", "functionCallOutput", "imageGeneration", "imageView", "mcpToolCall", "plan", "reasoning", "sleep", "subAgentActivity", "webSearch"} + +func isMessage(kind string) bool { return kind == "" || kind == userMessage } +func mayBeMessage(kind string) bool { return !slices.Contains(agentOutput, kind) } + +// ListingBounded: the bounded listing never reached the send (HostUnavailable). +type ListingBounded struct{ Message string } + +func (e *ListingBounded) Error() string { return e.Message } + +// ListingEmpty: the recipient listed no turns at all (HostUnavailable). +type ListingEmpty struct{ Message string } + +func (e *ListingEmpty) Error() string { return e.Message } + +// TurnPresence is hostadapter.TurnPresence. +type TurnPresence struct { + Finding string + Turn *TurnInfo + Scanned int + Stop string + Seen []any + Older []string + SeenTurns []TurnInfo + StopTurn *TurnInfo +} + +// ListingPage is one newest-first page of turns. +type ListingPage struct { + Turns []TurnInfo + Follows bool +} + +// Item is one thread item: (turn id, text, type); Type "" means untyped. +type Item struct{ Turn, Text, Type string } + +// ItemPage is one page of items. +type ItemPage struct { + Items []Item + Follows bool +} + +func olderTurns(turns []TurnInfo, cutoff float64) []string { + var out []string + for _, t := range turns { + if t.StartedAt != nil && *t.StartedAt <= cutoff { + out = append(out, t.TurnID) + } + } + return out +} + +// FindInListing is find_in_listing over pages already read. +func FindInListing(pages []ListingPage, turnID string, sentAt float64) (TurnPresence, error) { + next := 0 + return FindInListingPaged(func() (ListingPage, error) { + page := pages[next] + next++ + return page, nil + }, len(pages), turnID, sentAt) +} + +// FindInListingPaged is find_in_listing over a lazy listing: read yields the next page, at most +// bound pages are read, and nothing past the answer is read. turnID "" lists the turns since a +// send with no turn id; a listed turn without an id is never taken for it. +func FindInListingPaged(read func() (ListingPage, error), bound int, turnID string, sentAt float64) (TurnPresence, error) { + cutoff := sentAt - TurnStartPrecisionSeconds - DispatchTurnSkewSeconds + scanned := 0 + seen := []any{} + var seenTurns []TurnInfo + for p := 0; p < bound; p++ { + page, err := read() + if err != nil { + return TurnPresence{}, err + } + for i, turn := range page.Turns { + scanned++ + if turnID != "" && turn.TurnID == turnID { + match := turn + return TurnPresence{TurnPresent, &match, scanned, "matched", seen, olderAfterMatch(page.Turns[i+1:], page.Follows, read, bound-p-1, cutoff), seenTurns, nil}, nil + } + if turn.StartedAt != nil && *turn.StartedAt <= cutoff { + stop := turn + return TurnPresence{TurnAbsent, nil, scanned, "older_than_send", seen, olderTurns(page.Turns[i:], cutoff), seenTurns, &stop}, nil + } + seen = append(seen, idOf(turn)) + seenTurns = append(seenTurns, turn) + } + if !page.Follows { + if scanned == 0 { + return TurnPresence{}, &ListingEmpty{"the recipient's turn list is empty, which does not show that a turn is gone"} + } + return TurnPresence{TurnAbsent, nil, scanned, "listing_end", seen, nil, seenTurns, nil}, nil + } + } + subject := "the send's turn" + if turnID != "" { + subject = "turn " + pyReprValue(turnID) + } + return TurnPresence{}, &ListingBounded{fmt.Sprintf("%s was not among %d turns and the bounded listing never reached the send; this is not evidence of absence", subject, scanned)} +} + +// idOf is a listed turn's id, None when the host gave none. +func idOf(turn TurnInfo) any { + if turn.TurnID == "" { + return nil + } + return turn.TurnID +} + +// olderAfterMatch is _older_after_match: after a match, read on to the first turn begun before +// the send. A failed read or the page bound gives none; the match stands either way. +func olderAfterMatch(rest []TurnInfo, follows bool, read func() (ListingPage, error), left int, cutoff float64) []string { + for { + for i, turn := range rest { + if turn.StartedAt != nil && *turn.StartedAt <= cutoff { + return olderTurns(rest[i:], cutoff) + } + } + if !follows || left <= 0 { + return nil + } + page, err := read() + if err != nil { + return nil + } + left-- + rest, follows = page.Turns, page.Follows + } +} + +// FindTokenIn is find_token_in: a token among a thread's items since a send, newest first. +func FindTokenIn(pages []ItemPage, token string, older []string) TokenScan { + scanned := 0 + var other [2]any + for _, page := range pages { + for _, item := range page.Items { + if item.Turn != "" && slices.Contains(older, item.Turn) { + return TokenScan{false, nil, true, scanned, other[0], other[1]} + } + scanned++ + if !mayBeMessage(item.Type) || !strings.Contains(item.Text, token) { + continue + } + if isMessage(item.Type) { + return TokenScan{true, item.Turn, false, scanned, nil, nil} + } + if other[0] == nil && other[1] == nil { + other = [2]any{item.Turn, item.Type} + } + } + if !page.Follows { + return TokenScan{false, nil, true, scanned, other[0], other[1]} + } + } + return TokenScan{false, nil, false, scanned, other[0], other[1]} +} + +// FindTokenInTurnItems is find_token_in_turn_items. +func FindTokenInTurnItems(pages []ItemPage, token, turnID string) TokenScan { + scanned := 0 + var other [2]any + for _, page := range pages { + for _, item := range page.Items { + scanned++ + if item.Turn != turnID || !mayBeMessage(item.Type) || !strings.Contains(item.Text, token) { + continue + } + if isMessage(item.Type) { + return TokenScan{true, item.Turn, false, scanned, nil, nil} + } + if other[0] == nil && other[1] == nil { + other = [2]any{item.Turn, item.Type} + } + } + if !page.Follows { + return TokenScan{false, nil, true, scanned, other[0], other[1]} + } + } + return TokenScan{false, nil, false, scanned, other[0], other[1]} +} diff --git a/internal/relay/delivery/hostcheck.go b/internal/relay/delivery/hostcheck.go new file mode 100644 index 00000000..2bcf1749 --- /dev/null +++ b/internal/relay/delivery/hostcheck.go @@ -0,0 +1,422 @@ +package delivery + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The host-loss half of reconciliation (reconcile.py _settle_confirmed, check_dispatched_turn, +// confirm_delivery) and the daemon's recipient-turn pass (daemon._check_dispatched_turns). The +// rest of the tick is todo 29's; Turns is the state this pass keeps between ticks. + +// settleConfirmed is _settle_confirmed: an uncertain send already confirmed from its token stays +// confirmed, and what is left to ask is whether the parent still has that turn. +func (rc *Reconciler) settleConfirmed(ctx context.Context, attempt, delivery Row, observation string, adapter Adapter) (Obj, error) { + record := Obj{} + if !attempt.N("record") { + record = loadsObj(attempt.S("record")) + } + turnID, _ := get(record, "turnId") + if !truthy(turnID) { + turnID = delivery.Opt("dispatch_turn_id") + } + reading := ReadRecipientTurn(adapter, rc.Clock, attempt, delivery, turnID) + out := Obj{{Key: "evidence", Value: TurnFound}, {Key: "state", Value: attempt.Opt("state")}, {Key: "operationObservation", Value: observation}, {Key: "record", Value: record}, + {Key: "recipientTurn", Value: reading}, {Key: "detail", Value: "already confirmed from its token in the recipient's items; not scanned again"}} + return rc.afterReading(ctx, out, attempt, delivery, reading, observation) +} + +// afterReading records what a recipient-turn reading decided: a loss (completion) or a report +// only (revision), otherwise the undecided name. +func (rc *Reconciler) afterReading(ctx context.Context, out Obj, attempt, delivery Row, reading Reading, observation any) (Obj, error) { + switch { + case str(reading, "finding") == HostLostTurn && delivery.S("kind") == Completion: + loss, err := SettleLoss(ctx, rc.Store, rc.Clock, attempt.S("request_id"), reading, observation) + if err != nil { + return nil, err + } + for _, f := range loss { + out = set(out, f.Key, f.Value) + } + case str(reading, "finding") == HostLostTurn: + out = set(out, "redelivery", ReportOnly) + case delivery.S("kind") == Completion: + if _, err := RecordUndecided(ctx, rc.Store, attempt.S("request_id"), reading); err != nil { + return nil, err + } + } + return out, nil +} + +// CheckDispatchedTurn is check_dispatched_turn: the daemon's question for one delivered +// completion. Writes nothing unless the host lost the turn (or an undecided name changes). +func (rc *Reconciler) CheckDispatchedTurn(ctx context.Context, requestID string, adapter Adapter) (Obj, error) { + attempt, err := one(ctx, rc.Store, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return nil, err + } + if attempt == nil { + return nil, fmt.Errorf("KeyError: %s", store.PyRepr(requestID)) + } + delivery, err := rc.Delivery.Get(ctx, attempt.S("event_id")) + if err != nil { + return nil, err + } + record := Obj{} + if !attempt.N("record") { + record = loadsObj(attempt.S("record")) + } + turnID, _ := get(record, "turnId") + if !truthy(turnID) { + turnID = delivery.Opt("dispatch_turn_id") + } + reading := ReadRecipientTurn(adapter, rc.Clock, attempt, delivery, turnID) + out := Obj{{Key: "eventId", Value: attempt.S("event_id")}, {Key: "requestId", Value: requestID}, {Key: "state", Value: attempt.Opt("state")}, {Key: "recipientTurn", Value: reading}} + if str(reading, "finding") == HostLostTurn { + loss, err := SettleLoss(ctx, rc.Store, rc.Clock, requestID, reading, nil) + if err != nil { + return nil, err + } + for _, f := range loss { + out = set(out, f.Key, f.Value) + } + return out, nil + } + changed, err := RecordUndecided(ctx, rc.Store, requestID, reading) + if err != nil { + return nil, err + } + return append(out, F{Key: "undecidedChanged", Value: changed}), nil +} + +// ConfirmDelivery is confirm_delivery: settle an uncertain completion send before the parent's +// acknowledgement is judged, reading the acknowledging turn's own items for the message. Nil when +// nothing was read; read errors are reported in the outcome, never returned. +func (rc *Reconciler) ConfirmDelivery(ctx context.Context, eventID string, adapter Adapter, turnID string) (Obj, error) { + delivery, err := rc.Delivery.Find(ctx, eventID) + if err != nil || delivery == nil || delivery.S("kind") != Completion || delivery.S("state") != HeldUncertain { + return nil, err + } + current, err := one(ctx, rc.Store, "SELECT request_id FROM attempts WHERE event_id = ? AND attempt_no = ?", eventID, delivery.I("attempt_count")) + if err != nil || current == nil { + return nil, err + } + requestID := current.S("request_id") + out := Obj{{Key: "eventId", Value: eventID}, {Key: "requestId", Value: requestID}} + reconciled, err := rc.ReconcileAttempt(ctx, requestID, adapter, nil) + if err != nil { + return append(out, F{Key: "error", Value: "reconcile: " + errorLabel(err)}), nil + } + kept := Obj{} + for _, f := range reconciled { + if f.Key != "record" { + kept = append(kept, f) + } + } + out = append(out, F{Key: "reconciled", Value: kept}) + if turnID == "" { + return out, nil + } + if delivery, err = rc.Delivery.Find(ctx, eventID); err != nil { + return nil, err + } + attempt, err := one(ctx, rc.Store, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return nil, err + } + if delivery == nil || delivery.S("state") != HeldUncertain || delivery.I("attempt_count") != attempt.I("attempt_no") || attempt.S("internal_state") != "settled" || attempt.S("state") != HeldUncertain { + return out, nil + } + scan, err := adapter.FindTokenInTurn(delivery.S("recipient_thread_id"), requestID, turnID, InTurnItemsMax) + if err != nil { + return append(out, F{Key: "turnRead", Value: "unreadable: " + errorLabel(err)}), nil + } + detail := fmt.Sprintf("found=%s in turn %s, the acknowledging turn (%d of its items read)", pyStr(scan.Found), turnID, scan.Scanned) + out = append(out, F{Key: "turnRead", Value: detail}) + if !scan.Found { + return out, nil + } + observation := str(reconciled, "operationObservation") + if observation == "" { + observation = attempt.S("operation_observation") + } + settled, err := rc.settleFromScan(ctx, attempt, delivery, scan, observation, detail, rc.Clock.Now()) + switch { + case isRace(err): + name := "_AttemptChanged" + if errors.As(err, &attemptLost{}) { + name = "_AttemptLost" + } + return append(out, F{Key: "confirmed", Value: nil}, F{Key: "detail", Value: "the attempt was settled elsewhere first (" + name + ")"}), nil + case err != nil: + return nil, err + } + confirmed := Obj{} + for _, f := range settled { + if f.Key != "record" { + confirmed = append(confirmed, f) + } + } + return append(out, F{Key: "confirmed", Value: confirmed}), nil +} + +func isRace(err error) bool { + var changed attemptChanged + return errors.As(err, &attemptLost{}) || errors.As(err, &changed) +} + +// awaitingSQL is hostloss._AWAITING_SQL: delivered completions still owed an acknowledgement. +const awaitingSQL = "SELECT d.event_id, a.request_id FROM deliveries d JOIN attempts a ON a.event_id = d.event_id AND a.attempt_no = d.attempt_count WHERE d.kind = ? AND d.state = ? AND d.hold_reason IS NULL AND a.internal_state = 'settled' AND a.state IN (?, ?) AND NOT EXISTS (SELECT 1 FROM acks k WHERE k.event_id = d.event_id AND (k.verified = 'verified' OR a.sent_at IS NULL OR k.ack_at >= a.sent_at))" + +// AwaitingAck is hostloss.awaiting_ack: one page, keyed by event. +func AwaitingAck(ctx context.Context, s *store.Store, after string, limit int) ([]Row, error) { + query, args := awaitingSQL, []any{Completion, Dispatched, Dispatched, HeldUncertain} + if after != "" { + query += " AND d.event_id > ?" + args = append(args, after) + } + return all(ctx, s, query+" ORDER BY d.event_id LIMIT ?", append(args, limit)...) +} + +// StillAwaiting is hostloss.still_awaiting. +func StillAwaiting(ctx context.Context, s *store.Store, requestIDs []string) (map[string]bool, error) { + live := map[string]bool{} + if len(requestIDs) == 0 { + return live, nil + } + args := []any{Completion, Dispatched, Dispatched, HeldUncertain} + for _, id := range requestIDs { + args = append(args, id) + } + rows, err := all(ctx, s, awaitingSQL+" AND a.request_id IN ("+strings.TrimSuffix(strings.Repeat("?, ", len(requestIDs)), ", ")+")", args...) + for _, r := range rows { + live[r.S("request_id")] = true + } + return live, err +} + +// UndecidedRecheckSeconds is daemon.UNDECIDED_RECHECK_SECONDS. +const UndecidedRecheckSeconds = 600.0 + +// TurnCheckReport is the host-loss half of the daemon's TickReport. +type TurnCheckReport struct { + TurnsLost, TurnsUndecided int + Notes []string +} + +// TurnChecks is daemon._check_dispatched_turns with its in-memory state: where the rotation +// resumes, the finished turns not read again, and the undecided readings' next read. +type TurnChecks struct { + Reconciler *Reconciler + Budget int + Page int + after string + settled []string + undecided []string + nextRead map[string]float64 +} + +// Settled is the finished-turn memory, oldest checked first (daemon._turns_settled). +func (tc *TurnChecks) Settled() []string { return append([]string(nil), tc.settled...) } + +func remove(list []string, id string) []string { + for i, v := range list { + if v == id { + return append(list[:i:i], list[i+1:]...) + } + } + return list +} + +func (tc *TurnChecks) forgetDeparted(ctx context.Context, page int) error { + batch := append([]string(nil), tc.settled[:min(page, len(tc.settled))]...) + batch = append(batch, tc.undecided[:min(page, len(tc.undecided))]...) + if len(batch) == 0 { + return nil + } + live, err := StillAwaiting(ctx, tc.Reconciler.Store, batch) + if err != nil { + return err + } + for _, cache := range []*[]string{&tc.settled, &tc.undecided} { + for _, id := range batch { + if contains(*cache, id) { + *cache = remove(*cache, id) + if live[id] { + *cache = append(*cache, id) + } + } + } + } + for id := range tc.nextRead { + if !contains(tc.undecided, id) { + delete(tc.nextRead, id) + } + } + return nil +} + +func contains(list []string, id string) bool { + for _, v := range list { + if v == id { + return true + } + } + return false +} + +// Pass is one tick's recipient-turn check. +func (tc *TurnChecks) Pass(ctx context.Context, adapter Adapter, now float64, report *TurnCheckReport) { + budget := tc.Budget + if budget <= 0 || adapter == nil { + return + } + if tc.nextRead == nil { + tc.nextRead = map[string]float64{} + } + page := tc.Page + if page == 0 { + page = max(64, budget*16) + } + after := tc.after + if err := tc.forgetDeparted(ctx, page); err != nil { + report.Notes = append(report.Notes, "recipient turn check could not list deliveries: "+err.Error()) + return + } + rows, err := AwaitingAck(ctx, tc.Reconciler.Store, after, page) + if err == nil && after != "" && len(rows) < page { + var head []Row + if head, err = AwaitingAck(ctx, tc.Reconciler.Store, "", page); err == nil { + for _, r := range head { + if r.S("event_id") <= after { + rows = append(rows, r) + } + } + } + } + if err != nil { + report.Notes = append(report.Notes, "recipient turn check could not list deliveries: "+err.Error()) + return + } + spent := 0 + for _, row := range rows { + if spent >= budget { + break + } + tc.after = row.S("event_id") + id := row.S("request_id") + if contains(tc.settled, id) { + continue + } + if at, ok := tc.nextRead[id]; ok && contains(tc.undecided, id) && at > now { + continue + } + spent++ + outcome, err := tc.Reconciler.CheckDispatchedTurn(ctx, id, adapter) + if err != nil { + report.Notes = append(report.Notes, fmt.Sprintf("recipient turn check failed for %s: %s", id, err)) + continue + } + v, _ := get(outcome, "recipientTurn") + reading := v.(Obj) + if u, _ := get(reading, "undecided"); truthy(u) { + if !contains(tc.undecided, id) { + tc.undecided = append(tc.undecided, id) + } + tc.nextRead[id] = now + UndecidedRecheckSeconds + } else { + tc.undecided = remove(tc.undecided, id) + delete(tc.nextRead, id) + } + status, _ := get(reading, "status") + detail := str(reading, "detail") + if str(reading, "finding") == Present && contains(terminalTurn, pyStrOrEmpty(status)) { + if !contains(tc.settled, id) { + tc.settled = append(tc.settled, id) + } + } else if str(reading, "finding") == Unknown && strings.HasPrefix(detail, "unreadable") { + report.Notes = append(report.Notes, fmt.Sprintf("recipient turn unreadable for %s: %s", id, detail)) + } + if r := str(outcome, "redelivery"); r == Requeued || r == HeldRedelivery { + report.TurnsLost++ + } + if n, ok := get(outcome, "undecidedChanged"); ok { + report.TurnsUndecided += int(n.(int64)) + } + } +} + +// KeptUnconfirmed is kept_unconfirmed: (event, acknowledging turn) of each acknowledgement kept +// while its delivery is still unconfirmed and whose check is due. +func (a *Ack) KeptUnconfirmed(ctx context.Context, now float64, limit int) ([][2]string, error) { + rows, err := all(ctx, a.Store, "SELECT a.event_id, a.ack_turn_id FROM acks a JOIN ack_evidence e ON e.event_id = a.event_id JOIN deliveries d ON d.event_id = a.event_id WHERE a.verified = 'unverified_turn' AND e.last_reason = ? AND d.state = ? AND (e.next_check_at IS NULL OR e.next_check_at <= ?) ORDER BY COALESCE(e.next_check_at, 0), a.event_id LIMIT ?", + DeliveryUnconfirmed, HeldUncertain, now, limit) + out := make([][2]string, 0, len(rows)) + for _, r := range rows { + out = append(out, [2]string{r.S("event_id"), r.S("ack_turn_id")}) + } + return out, err +} + +func (a *Ack) kept(ctx context.Context, eventID string) (Row, error) { + return one(ctx, a.Store, "SELECT a.event_id, a.ack_turn_id, a.ack_at, a.accepted, COALESCE(e.attempts, 0) AS attempts, e.last_reason, e.fingerprint, e.next_check_at FROM acks a JOIN ack_evidence e ON e.event_id = a.event_id WHERE a.event_id = ? AND a.verified = 'unverified_turn' AND e.last_reason = ?", eventID, DeliveryUnconfirmed) +} + +// KeptTurn is kept_turn: the acknowledging turn of a kept acknowledgement, or "". +func (a *Ack) KeptTurn(ctx context.Context, eventID string) (string, error) { + row, err := a.kept(ctx, eventID) + if err != nil || row == nil { + return "", err + } + return row.S("ack_turn_id"), nil +} + +// CompletePending is complete_pending: one kept acknowledgement completed now, as the pending +// pass would. Nil when nothing is kept for this event. +func (a *Ack) CompletePending(ctx context.Context, eventID string, adapter Adapter) (Obj, error) { + now := a.Clock.Now() + pending, err := a.kept(ctx, eventID) + if err != nil || pending == nil { + return nil, err + } + row, err := a.Delivery.Find(ctx, eventID) + if err != nil || row == nil { + return nil, err + } + verification, err := a.verifyAckTurn(ctx, row, pending.S("ack_turn_id"), adapter) + if err != nil { + if Reason(err) == "" { + return nil, err + } + verification = Reason(err) + } + return a.settlePendingAck(ctx, eventID, pending, verification, now, row) +} + +// ConfirmKeptAcks is daemon._confirm_kept_acks. +func ConfirmKeptAcks(ctx context.Context, a *Ack, rc *Reconciler, adapter Adapter, now float64) []string { + var notes []string + rows, err := a.KeptUnconfirmed(ctx, now, 8) + if err != nil { + return []string{"kept acknowledgement pass failed: " + err.Error()} + } + for _, r := range rows { + outcome, err := rc.ConfirmDelivery(ctx, r[0], adapter, r[1]) + if err != nil { + notes = append(notes, fmt.Sprintf("kept acknowledgement %s not confirmed: %s", r[0], err)) + continue + } + problem := str(outcome, "error") + if read := str(outcome, "turnRead"); problem == "" && strings.HasPrefix(read, "unreadable") { + problem = read + } + if problem != "" { + notes = append(notes, fmt.Sprintf("kept acknowledgement %s not confirmed: %s", r[0], problem)) + } + } + return notes +} diff --git a/internal/relay/delivery/hostloss.go b/internal/relay/delivery/hostloss.go new file mode 100644 index 00000000..29f0e619 --- /dev/null +++ b/internal/relay/delivery/hostloss.go @@ -0,0 +1,336 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Host-loss words (hostloss.py). +const ( + Present = "present" + Unknown = "unknown" + tokenScanLimit = 200 + Requeued = "queued" + HeldRedelivery = "held" + NotMoved = "not_moved" + ReportOnly = "report_only" + ListingBoundedW = "listing_bounded" + ListingEmptyW = "listing_empty" + TokenScanBounded = "token_scan_bounded" + TokenWithoutTurn = "token_without_turn" + TokenInOtherItem = "token_in_other_item" + NoSendTime = "no_send_time" + NoTurn = "no_turn" + ReceiptUnsettled = "receipt_unsettled" + ReceiptMissing = "receipt_missing" + SettledReceipt = "settled" + UnsettledReceipt = "unsettled" + MissingReceipt = "missing" + undecidedMark = TurnCheckUndecided + ":" + unknownLostMark = UnknownSendLost + ":no_trace" + unknownUndecided = UnknownSendUndecided + ":" + unknownMarkPrefix = "unknown_send_" +) + +var terminalTurn = []string{"completed", "interrupted", "failed"} +var deliveredAttemptStates = []string{Dispatched, HeldUncertain} + +func epoch(stamp string) (float64, bool) { + t, err := time.Parse("2006-01-02T15:04:05.999999-07:00", stamp) + if err != nil { + return 0, false + } + return float64(t.UnixMicro()) / 1e6, true +} + +// Reading is what a recipient's own turns and items say about an attempt. +type Reading = Obj + +func newReading(turnID any) Reading { + return Obj{{Key: "turnId", Value: turnID}, {Key: "finding", Value: Unknown}, {Key: "status", Value: nil}, {Key: "detail", Value: nil}, {Key: "undecided", Value: nil}} +} + +func allowance() float64 { return TurnStartPrecisionSeconds + DispatchTurnSkewSeconds } + +// ReadRecipientTurn is hostloss.read_recipient_turn. Reads only. +func ReadRecipientTurn(adapter Adapter, clock Clock, attempt, delivery Row, turnID any) Reading { + r := newReading(turnID) + thread := delivery.S("recipient_thread_id") + turn, _ := turnID.(string) + if turn == "" { + return set(set(r, "detail", "the attempt names no turn"), "undecided", NoTurn) + } + sentAt, ok := epoch(attempt.S("sent_at")) + if !ok { + return set(set(r, "detail", "the attempt has no send time, so absence cannot be bounded"), "undecided", NoSendTime) + } + allow := allowance() + presence, err := adapter.FindDispatchedTurn(thread, turn, sentAt) + var bounded *ListingBounded + var empty *ListingEmpty + switch { + case errors.As(err, &bounded): + return set(set(r, "detail", "undecided: "+bounded.Message), "undecided", ListingBoundedW) + case errors.As(err, &empty): + if clock.Now() < sentAt+allow { + return set(r, "detail", fmt.Sprintf("the recipient lists no turns yet, and the send is less than %.0f s old: %s", allow, empty.Message)) + } + return set(set(r, "detail", "undecided: "+empty.Message), "undecided", ListingEmptyW) + case err != nil: + return set(r, "detail", "unreadable: "+errorLabel(err)) + } + listed := presence.Finding == TurnPresent + where := "does not list this turn" + if listed { + status := presence.Turn.Status + if !slices.Contains(terminalTurn, status) { + return set(set(set(r, "finding", Present), "status", status), "detail", fmt.Sprintf("the recipient lists this turn (%s)", status)) + } + own, err := adapter.FindTokenInTurn(thread, attempt.S("request_id"), turn, InTurnItemsMax) + if err != nil { + return set(r, "detail", fmt.Sprintf("unreadable: the recipient lists this turn (%s), but its items could not be read for this attempt's message: %s", status, errorLabel(err))) + } + if own.Found { + return set(set(set(r, "finding", Present), "status", status), "detail", fmt.Sprintf("the recipient lists this turn (%s) with this attempt's message in it", status)) + } + where = fmt.Sprintf("lists this turn %s without this attempt's message among its first %d items", status, own.Scanned) + } + if clock.Now() < sentAt+allow { + if listed { + return set(r, "detail", fmt.Sprintf("the recipient %s, but the send is less than %.0f s old, too recent to call the turn lost", where, allow)) + } + return set(r, "detail", fmt.Sprintf("the recipient does not list this turn yet (%s), but the send is less than %.0f s old, too recent to call the turn lost", presence.Stop, allow)) + } + scan, err := adapter.FindTokenSince(thread, attempt.S("request_id"), presence.Older, tokenScanLimit) + if err != nil { + return set(r, "detail", fmt.Sprintf("the recipient %s, and its items could not be read for this attempt's token: %s", where, errorLabel(err))) + } + if scan.Found { + if listed { + return set(set(r, "finding", Present), "detail", fmt.Sprintf("the recipient %s, but this attempt's message is in its items (turn %s)", where, pyStr(scan.TurnID))) + } + return set(set(set(r, "finding", Present), "detail", fmt.Sprintf("the recipient does not list this turn, but this attempt's token is in its items (turn %s)", pyStr(scan.TurnID))), "undecided", TokenWithoutTurn) + } + if scan.OtherKind != nil { + return set(set(r, "detail", fmt.Sprintf("undecided: the recipient %s, and this attempt's token is in its items only in an item of type %s (turn %s), which is neither the delivered message nor agent output; not sent again", where, pyStr(scan.OtherKind), pyStr(scan.OtherTurn))), "undecided", TokenInOtherItem) + } + if !scan.Exhausted { + return set(set(r, "detail", fmt.Sprintf("undecided: the recipient %s, and %d items did not reach history older than the send, so the token's absence is not shown", where, scan.Scanned)), "undecided", TokenScanBounded) + } + if listed { + return set(set(r, "finding", HostLostTurn), "detail", fmt.Sprintf("the recipient %s, and this attempt's token is not among the %d items since the send (%d listed turns begun since it): the host lost the turn's content", where, scan.Scanned, len(presence.Seen))) + } + return set(set(r, "finding", HostLostTurn), "detail", fmt.Sprintf("the recipient's turn list has no such turn (%s after %d turns) and this attempt's token is not among the %d items since the send (%d listed turns begun since it)", presence.Stop, presence.Scanned, scan.Scanned, len(presence.Seen))) +} + +// RecordUndecided is hostloss.record_undecided; returns the rows changed. +func RecordUndecided(ctx context.Context, s *store.Store, requestID string, reading Reading) (int64, error) { + var wanted any + if u, _ := get(reading, "undecided"); truthy(u) { + wanted = undecidedMark + pyStr(u) + } else if str(reading, "finding") != Present { + return 0, nil + } + var changed int64 + err := s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + row, err := one(ctx, s, "SELECT internal_state, state, recipient_scan FROM attempts WHERE request_id = ?", requestID) + if err != nil || row == nil || row.S("internal_state") != "settled" || !slices.Contains(deliveredAttemptStates, row.S("state")) { + return err + } + current := row.S("recipient_scan") + if wanted == nil && !strings.HasPrefix(current, undecidedMark) { + return nil + } + if wanted != nil && !row.N("recipient_scan") && current == wanted { + return nil + } + changed, err = execSQL(ctx, s, "UPDATE attempts SET recipient_scan = ? WHERE request_id = ?", wanted, requestID) + return err + }) + return changed, err +} + +var errRaced = errors.New("raced") + +// SettleLoss is hostloss.settle: record the loss and put the obligation back, once. +func SettleLoss(ctx context.Context, s *store.Store, clock Clock, requestID string, reading Reading, observation any) (Obj, error) { + now := clock.ISO() + var record Obj + var hold any + notMoved := false + err := s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + attempt, err := one(ctx, s, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return err + } + earlier, err := one(ctx, s, "SELECT COUNT(*) AS c FROM attempts WHERE event_id = ? AND state = ? AND request_id <> ?", attempt.S("event_id"), HostLostTurn, requestID) + if err != nil { + return err + } + if earlier.I("c") > 0 { + hold = HostLostTurn + } + moved, err := execSQL(ctx, s, "UPDATE deliveries SET state = ?, hold_reason = ?, next_eligible_at = NULL, dispatch_evidence = ?, lease_owner = NULL, lease_until = NULL, updated_at = ? WHERE event_id = ? AND kind = ? AND state = ? AND attempt_count = ? AND hold_reason IS NULL AND NOT EXISTS (SELECT 1 FROM acks k WHERE k.event_id = deliveries.event_id AND (k.verified = 'verified' OR ? IS NULL OR k.ack_at >= ?))", + Queued, hold, HostLostTurn, now, attempt.S("event_id"), Completion, Dispatched, attempt.I("attempt_no"), attempt.Opt("sent_at"), attempt.Opt("sent_at")) + if err != nil { + return err + } + if moved != 1 { + notMoved = true + return nil + } + record = loadsObj(attempt.S("record")) + evidence := attempt.S("affirmative_evidence") + if evidence == "" { + evidence = "receipt_turn_id" + } + record = set(record, "reconciliation", Obj{{Key: "operationReceiptChecked", Value: observation != nil}, {Key: "recipientTurnsChecked", Value: true}, {Key: "affirmativeEvidence", Value: evidence}, {Key: "checkedAt", Value: now}}) + if err := AssertAttemptInvariants(record); err != nil { + return err + } + marked, err := execSQL(ctx, s, "UPDATE attempts SET state = ?, record = ?, operation_observation = COALESCE(?, operation_observation), recipient_scan = ?, affirmative_evidence = ?, reconciled_at = ? WHERE request_id = ? AND internal_state = 'settled' AND state IN (?, ?)", + HostLostTurn, dumps(record), observation, str(reading, "detail"), evidence, now, requestID, Dispatched, HeldUncertain) + if err != nil { + return err + } + if marked != 1 { + return errRaced + } + redelivery := Requeued + if hold != nil { + redelivery = HeldRedelivery + } + turnID, _ := get(reading, "turnId") + return journal(ctx, s, HostLostTurn, attempt.S("event_id"), Obj{{Key: "requestId", Value: requestID}, {Key: "turnId", Value: turnID}, {Key: "redelivery", Value: redelivery}, {Key: "detail", Value: str(reading, "detail")}}, now) + }) + if errors.Is(err, errRaced) { + return Obj{{Key: "redelivery", Value: NotMoved}, {Key: "redeliveryDetail", Value: "the attempt changed while the loss was being recorded"}}, nil + } + if err != nil { + return nil, err + } + if notMoved { + return Obj{{Key: "redelivery", Value: NotMoved}, {Key: "redeliveryDetail", Value: "the delivery is no longer this attempt's unheld, unacknowledged dispatch"}}, nil + } + redelivery := Requeued + if hold != nil { + redelivery = HeldRedelivery + } + return Obj{{Key: "state", Value: HostLostTurn}, {Key: "redelivery", Value: redelivery}, {Key: "holdReason", Value: hold}, {Key: "record", Value: record}}, nil +} + +func foldCandidates(p TurnPresence, sentAt float64) []TurnInfo { + var out []TurnInfo + for _, t := range p.SeenTurns { + if t.StartedAt != nil && *t.StartedAt <= sentAt+TurnStartPrecisionSeconds { + out = append(out, t) + } + } + if p.StopTurn != nil { + out = append(out, *p.StopTurn) + } + return out +} + +// ReadUnknownSend is hostloss.read_unknown_send. Reads only; never allows a second send. +func ReadUnknownSend(adapter Adapter, clock Clock, attempt, delivery Row, receipt string) Reading { + r := Obj{{Key: "turnId", Value: nil}, {Key: "finding", Value: Unknown}, {Key: "detail", Value: nil}, {Key: "undecided", Value: nil}, {Key: "pending", Value: false}} + thread := delivery.S("recipient_thread_id") + request := attempt.S("request_id") + sentAt, ok := epoch(attempt.S("sent_at")) + if !ok { + return set(set(r, "detail", "the attempt has no send time, so absence cannot be bounded"), "undecided", NoSendTime) + } + allow := allowance() + pending := func(detail string) Reading { return set(set(r, "detail", detail), "pending", true) } + undecided := func(detail, why string) Reading { return set(set(r, "detail", detail), "undecided", why) } + if clock.Now() < sentAt+allow { + return pending(fmt.Sprintf("the send is less than %.0f s old, too recent to call it lost; read again", allow)) + } + presence, err := adapter.FindDispatchedTurn(thread, "", sentAt) + var bounded *ListingBounded + var empty *ListingEmpty + switch { + case errors.As(err, &bounded): + return undecided("undecided: "+bounded.Message, ListingBoundedW) + case errors.As(err, &empty): + return undecided("undecided: "+empty.Message, ListingEmptyW) + case err != nil: + return pending("unreadable: " + errorLabel(err)) + } + folded := foldCandidates(presence, sentAt) + var running []string + for _, t := range presence.SeenTurns { + if !slices.Contains(terminalTurn, t.Status) { + running = append(running, t.TurnID) + } + } + if presence.StopTurn != nil && !slices.Contains(terminalTurn, presence.StopTurn.Status) { + running = append(running, presence.StopTurn.TurnID) + } + if len(running) > 0 { + return pending(fmt.Sprintf("a turn begun since the send is still running (%s); read again once it ends", running[0])) + } + scan, err := adapter.FindTokenSince(thread, request, presence.Older, tokenScanLimit) + if err != nil { + return pending("unreadable: the recipient's items could not be read for this attempt's token: " + errorLabel(err)) + } + if scan.Found { + return set(set(set(r, "finding", Present), "turnId", scan.TurnID), "detail", fmt.Sprintf("this attempt's message is in the recipient's items since the send (turn %s, %d items read)", pyStr(scan.TurnID), scan.Scanned)) + } + type owned struct { + turn TurnInfo + scan TokenScan + } + var owns []owned + for _, turn := range folded { + own, err := adapter.FindTokenInTurn(thread, request, turn.TurnID, InTurnItemsMax) + if err != nil { + return pending(fmt.Sprintf("unreadable: the items of turn %s, one the send could have been folded into, could not be read: %s", turn.TurnID, errorLabel(err))) + } + if own.Found { + return set(set(set(r, "finding", Present), "turnId", turn.TurnID), "detail", fmt.Sprintf("this attempt's message is in turn %s, begun by the send, which the send was folded into", turn.TurnID)) + } + owns = append(owns, owned{turn, own}) + } + for _, o := range owns { + if o.scan.OtherKind != nil { + return undecided(fmt.Sprintf("undecided: this attempt's token is in turn %s, which the send could have been folded into, only in an item of type %s, which is neither the delivered message nor agent output; not sent again", o.turn.TurnID, pyStr(o.scan.OtherKind)), TokenInOtherItem) + } + } + if scan.OtherKind != nil { + return undecided(fmt.Sprintf("undecided: this attempt's token is in the recipient's items only in an item of type %s (turn %s), which is neither the delivered message nor agent output; not sent again", pyStr(scan.OtherKind), pyStr(scan.OtherTurn)), TokenInOtherItem) + } + if !scan.Exhausted { + return undecided(fmt.Sprintf("undecided: %d items did not reach history older than the send, so the token's absence is not shown", scan.Scanned), TokenScanBounded) + } + for _, o := range owns { + if !o.scan.Exhausted { + return undecided(fmt.Sprintf("undecided: turn %s, begun by the send, could have taken it, and its first %d items did not reach its end", o.turn.TurnID, o.scan.Scanned), TokenScanBounded) + } + } + if receipt != SettledReceipt { + said := "the transport has not settled this request's receipt (in_progress_or_unknown)" + why := ReceiptUnsettled + if receipt == MissingReceipt { + said, why = "the transport holds no receipt for this request id", ReceiptMissing + } + return undecided("undecided: the recipient keeps no trace of this send, but "+said, why) + } + foldedText := "" + if len(folded) > 0 { + ids := make([]string, len(folded)) + for i, t := range folded { + ids[i] = t.TurnID + } + foldedText = " nor in " + strings.Join(ids, ", ") + ", the turns begun by it" + } + return set(set(r, "finding", UnknownSendLost), "detail", fmt.Sprintf("the recipient keeps no trace of this send: its turn list (%s after %d turns) shows %d turns begun since it, none running, and this attempt's token is not among the %d items since it%s; held for the parent, never sent again", presence.Stop, presence.Scanned, len(presence.Seen), scan.Scanned, foldedText)) +} diff --git a/internal/relay/delivery/hostloss_a_test.go b/internal/relay/delivery/hostloss_a_test.go new file mode 100644 index 00000000..981d113b --- /dev/null +++ b/internal/relay/delivery/hostloss_a_test.go @@ -0,0 +1,364 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +// test_host_lost_turn.py HLT-1..HLT-29. Each Test21_HLT runs the Go twin of every Python test +// the property lists (as a subtest named after it), in the Python test's own tree, and compares +// every asserted value, the delivery tables and the sends with what Python produced +// (hostloss_harness_test.go). + +const hlt = "test_host_lost_turn" + +func Test21_HLT01_a_lost_accepted_turn_is_redelivered_once_under_the_next_attempt(t *testing.T) { + t.Run("turn missing from the list", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_the_daemon_redelivers_the_same_event_once_under_the_next_attempt", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + report := h.tick() + attempts := h.attemptsFor(event) + h.eq(h.attemptStates(event)) + h.eq([]any{attempts[0].S("event_id"), attempts[1].S("event_id")}) + h.eq(sendIDs(h)) + h.eq(first) + h.eq(field(report.asDict(), "turnsLost")) + h.eq(h.journalled(HostLostTurn)) + h.eq(field(h.statusOf(event), "phase")) + for i := 0; i < 3; i++ { + h.clock.Advance(120) + h.tick() + } + h.eq(len(h.host.sends)) + h.eq(h.journalled(HostLostTurn)) + }) + }) + t.Run("turn in progress then missing", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_turn_first_read_in_progress_and_lost_later_is_still_caught", func(h *hl) { + event, _, turn := h.dispatched() + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.hostLoses(turn, true) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)[0]) + }) + }) + t.Run("turn relisted interrupted without its message", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_turn_reloaded_interrupted_without_its_message_is_lost_and_redelivered_once", func(h *hl) { + event, first, turn := h.dispatched() + h.hostReloadsLosing(turn) + h.clock.Advance(5) + h.tick() + h.eq(h.attemptStates(event)) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(h.journalled(HostLostTurn)) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq(field(reading, "finding")) + h.eq(strings.Contains(str(reading, "detail"), "interrupted")) + }) + }) +} + +func sendIDs(h *hl) []any { + out := []any{} + for _, s := range h.host.sends { + out = append(out, s.requestID) + } + return out +} + +func Test21_HLT02_a_second_loss_holds_the_obligation_under_its_name(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_second_loss_holds_the_obligation_under_its_name_instead_of_a_third_send", func(h *hl) { + event, _, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.tick() + second := h.attemptsFor(event)[1] + h.hostLoses(str(loadsObj(second.S("record")), "turnId"), true) + for i := 0; i < 4; i++ { + h.clock.Advance(120) + h.tick() + } + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(h.row(event).Opt("hold_reason")) + h.eq(h.statusPair(event)) + }) +} + +func Test21_HLT03_a_token_confirmed_completion_is_checked_like_an_accepted_one(t *testing.T) { + t.Run("checked for host loss", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_completion_confirmed_by_its_token_is_checked_like_an_accepted_one", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + turn := h.host.startTurn(parent, "", "completed", "..."+request+"...") + h.eq(str(h.reconcile(request, h.host), "evidence")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.hostLoses(turn.TurnID, true) + h.clock.Advance(120) + h.eq(field(h.tick().asDict(), "turnsLost")) + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + lost := loadsObj(h.attemptsFor(event)[0].S("record")) + h.eq(field(sub(lost, "reconciliation"), "affirmativeEvidence")) + h.eq(field(lost, "deliveryState")) + for i := 0; i < 3; i++ { + h.clock.Advance(120) + h.tick() + } + h.eq(len(h.host.sends)) + }) + }) + t.Run("a manual reconcile keeps turn_found", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_manual_reconcile_keeps_a_token_confirmed_send_confirmed", func(h *hl) { + event, request, turn := h.tokenConfirmed() + h.hostLoses(turn, true) + h.clock.Advance(5) + h.eq(field(h.reconcile(request, h.host), "evidence")) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(h.evidenceOf(event)[0]) + h.eq(field(sub(loadsObj(h.attemptsFor(event)[0].S("record")), "reconciliation"), "affirmativeEvidence")) + }) + }) + t.Run("a recorded loss reports its own evidence", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_reconciling_a_lost_token_confirmed_attempt_reports_its_own_evidence", func(h *hl) { + event, request, turn := h.tokenConfirmed() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)[0]) + again := h.reconcile(request, h.host) + h.eq([]any{field(again, "state"), field(again, "evidence")}) + h.eq(len(h.host.sends)) + }) + }) +} + +func Test21_HLT05_a_tick_whose_only_change_is_the_loss_is_not_quiet(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_tick_whose_only_change_is_the_loss_is_not_quiet", func(h *hl) { + event, _, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + policy := defaultTick() + policy.maxSendsTick = 0 + report := h.tickWith(policy, h.host, h.checks) + counters := report.asDict() + h.eq(field(counters, "turnsLost")) + others := map[string]any{} + for _, f := range counters { + if f.Key != "turnsLost" && f.Key != "notes" && f.Key != "skipped" && f.Key != "quiet" { + others[f.Key] = f.Value + } + } + h.eq(others) + h.eq(report.quiet()) + h.eq(h.statusPair(event)) + detail := field(h.statusOf(event), "attemptDetail").([]any) + h.eq(detail[0].(Row).S("state")) + messages, err := h.delivery.AttemptMessages(h.ctx, event) + mustDo(t, err) + statuses := []any{} + for _, m := range messages { + statuses = append(statuses, field(m.(Obj), "status")) + } + h.eq(statuses) + }) +} + +func Test21_HLT06_a_send_too_recent_to_judge_is_unknown_until_the_allowance_has_passed(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_send_too_recent_to_judge_is_unknown_until_the_allowance_has_passed", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(5) + h.eq(field(sub(h.reconcile(first, h.host), "recipientTurn"), "finding")) + h.eq(h.row(event).S("state")) + h.clock.Advance(120) + h.eq(field(sub(h.reconcile(first, h.host), "recipientTurn"), "finding")) + }) +} + +func Test21_HLT07_the_turn_check_budget_reaches_every_delivery_and_reads_finished_turns_once(t *testing.T) { + t.Run("budget 2 per tick over 5 deliveries", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_the_budget_bounds_lookups_and_every_delivery_is_reached", func(h *hl) { + h.parentHistory() + var turns []string + for i := 0; i < 5; i++ { + rid := h.register(regOpts{issue: fmt.Sprintf("REL-%d", i+2), dispatchRequest: fmt.Sprintf("dispatch-%d", i+2)}) + h.rid = rid + payload := h.readyPayload(rid, 1, []string{h.artifact(fmt.Sprintf("out-%d.txt", i), fmt.Sprintf("deliverable %d", i))}, 1, assigned("completed")) + _, err := h.accept(payload, storeAcceptNone) + mustDo(t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(t, err) + record := h.attemptOn(str(payload, "eventId"), h.host, nil) + h.eq(str(record, "deliveryState")) + h.eq(field(record, "attemptNo")) + turns = append(turns, str(record, "turnId")) + h.clock.Advance(10) + } + for _, turn := range turns[:2] { + h.host.finishTurn(parent, turn, "completed") + } + var lookups []string + counting := countingLookups(h.host, &lookups) + policy := tickPolicy{2, 0, 8} + checks := &TurnChecks{Reconciler: h.rc} + var perTick []any + for i := 0; i < 3; i++ { + before := len(lookups) + h.tickWith(policy, counting, checks) + perTick = append(perTick, len(lookups)-before) + } + h.eq(perTick) + h.eq(sortedSet(lookups)) + for i := 0; i < 3; i++ { + h.tickWith(policy, counting, checks) + } + h.eq([]any{countOf(lookups, turns[0]), countOf(lookups, turns[1])}) + ok := true + for _, turn := range turns[2:] { + ok = ok && countOf(lookups, turn) > 1 + } + h.eq(ok) + h.eq(len(h.host.sends)) + }) + }) + t.Run("finished turns read once across pages", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_finished_turn_is_read_once_even_when_the_candidates_span_pages", func(h *hl) { + h.parentHistory() + var turns []string + for i := 0; i < 5; i++ { + rid := h.register(regOpts{issue: fmt.Sprintf("REL-%d", i+2), dispatchRequest: fmt.Sprintf("dispatch-%d", i+2)}) + h.rid = rid + payload := h.readyPayload(rid, 1, []string{h.artifact(fmt.Sprintf("out-%d.txt", i), fmt.Sprintf("deliverable %d", i))}, 1, assigned("completed")) + _, err := h.accept(payload, storeAcceptNone) + mustDo(t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(t, err) + turns = append(turns, str(h.attemptOn(str(payload, "eventId"), h.host, nil), "turnId")) + h.clock.Advance(10) + } + for _, turn := range turns { + h.host.finishTurn(parent, turn, "completed") + } + var lookups []string + counting := countingLookups(h.host, &lookups) + checks := &TurnChecks{Reconciler: h.rc, Page: 2} + for i := 0; i < 8; i++ { + h.tickWith(tickPolicy{2, 0, 8}, counting, checks) + } + h.eq(sortedList(lookups)) + }) + }) + t.Run("a loss behind two pages of finished turns", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_delivery_behind_two_pages_of_finished_turns_is_still_reached", func(h *hl) { + delivered := h.completions(5) + for _, d := range delivered[:4] { + h.host.finishTurn(parent, d[2], "completed") + } + lostEvent, lostTurn := delivered[4][0], delivered[4][2] + h.hostLoses(lostTurn, true) + h.clock.Advance(120) + checks := &TurnChecks{Reconciler: h.rc, Page: 2} + for i := 0; i < 6; i++ { + h.tickWith(tickPolicy{4, 4, 8}, h.host, checks) + h.clock.Advance(20) + } + h.eq(h.attemptStates(lostEvent)) + h.eq(len(h.host.sends)) + }) + }) + t.Run("the finished-turn memory is pruned", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_the_finished_turns_remembered_are_only_those_still_awaiting", func(h *hl) { + delivered := h.completions(5) + for _, d := range delivered { + h.host.finishTurn(parent, d[2], "completed") + } + h.clock.Advance(120) + checks := &TurnChecks{Reconciler: h.rc, Page: 2} + policy := tickPolicy{4, 0, 8} + for i := 0; i < 4; i++ { + h.tickWith(policy, h.host, checks) + } + acknowledged := []string{delivered[0][1], delivered[1][1], delivered[2][1]} + subset := true + for _, id := range acknowledged { + subset = subset && contains(checks.Settled(), id) + } + h.eq(subset) + for i, d := range delivered[:3] { + h.acknowledge(d[0], fmt.Sprintf("ack-%d", i)) + } + for i := 0; i < 4; i++ { + h.clock.Advance(20) + h.tickWith(policy, h.host, checks) + } + left := []string{} + for _, id := range acknowledged { + if contains(checks.Settled(), id) { + left = append(left, id) + } + } + h.eq(sortedSet(left)) + h.eq(len(h.host.sends)) + }) + }) +} + +func countOf(list []string, v string) int { + n := 0 + for _, x := range list { + if x == v { + n++ + } + } + return n +} + +// sortedSet is plain(set(...)) in capture.py: sorted by JSON text, duplicates dropped. +func sortedSet(list []string) []any { + seen := map[string]bool{} + var uniq []string + for _, v := range list { + if !seen[v] { + seen[v] = true + uniq = append(uniq, v) + } + } + return sortedList(uniq) +} + +func sortedList(list []string) []any { + keyed := make([]string, len(list)) + copy(keyed, list) + for i := range keyed { + for j := i + 1; j < len(keyed); j++ { + a, _ := json.Marshal(keyed[i]) + b, _ := json.Marshal(keyed[j]) + if string(b) < string(a) { + keyed[i], keyed[j] = keyed[j], keyed[i] + } + } + } + out := make([]any, len(keyed)) + for i, v := range keyed { + out[i] = v + } + return out +} diff --git a/internal/relay/delivery/hostloss_b_test.go b/internal/relay/delivery/hostloss_b_test.go new file mode 100644 index 00000000..443250c8 --- /dev/null +++ b/internal/relay/delivery/hostloss_b_test.go @@ -0,0 +1,590 @@ +package delivery + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" +) + +// test_host_lost_turn.py HLT-4, HLT-8..HLT-17 (see hostloss_a_test.go for the method). + +func Test21_HLT04_an_undecided_turn_check_is_named_and_kept(t *testing.T) { + t.Run("re-confirm from the token keeps the name", func(t *testing.T) { + mirror(t, hlt, "TheHostLostTheAcceptedTurn.test_a_manual_reconcile_keeps_the_name_on_a_token_confirmed_send", func(h *hl) { + event, request, turn := h.tokenConfirmed() + h.hostLoses(turn, false) + h.clock.Advance(120) + h.tick() + h.eq(field(h.completionDelivery(), "turnCheck")) + outcome := h.reconcile(request, h.host) + h.eq(field(outcome, "evidence")) + h.eq(field(sub(outcome, "recipientTurn"), "undecided")) + h.eq(field(h.completionDelivery(), "turnCheck")) + h.eq(field(h.statusOf(event), "phase")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("token_without_turn", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_delivery_turn_gone_from_the_list_with_its_token_kept_is_named", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, false) + h.clock.Advance(120) + h.eq(field(h.tick().asDict(), "turnsUndecided")) + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(field(h.statusOf(event), "phase")) + h.eq(field(h.completionDelivery(), "turnCheck")) + outcome := h.reconcile(first, h.host) + reading := sub(outcome, "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "undecided")}) + h.eq(field(sub(sub(outcome, "record"), "reconciliation"), "recipientTurnsChecked")) + h.eq(field(h.completionDelivery(), "turnCheck")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("token deeper than the scan", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_token_deeper_than_the_scan_in_a_later_turn_is_undecided_not_lost", func(h *hl) { + event, first, turn := h.dispatched() + message := h.messageOf(turn) + h.hostLoses(turn, true) + h.host.startTurn(parent, "parent-later", "completed", "") + h.item(parent, "parent-later", message, "userMessage") + for n := 0; n < 201; n++ { + h.item(parent, "parent-later", fmt.Sprintf("work item %d", n), "userMessage") + } + h.clock.Advance(120) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "undecided")}) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(field(h.statusOf(event), "phase")) + h.eq(field(h.completionDelivery(), "turnCheck")) + }) + }) + t.Run("listing bounded before the send", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_listing_too_long_to_reach_the_send_is_recorded_as_undecided", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + never := &hooked{Adapter: h.host, findDispatched: func(string, string, float64) (TurnPresence, error) { + return TurnPresence{}, &ListingBounded{"1000 newer turns and the send not reached"} + }} + reading := sub(h.reconcile(first, never), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "undecided")}) + h.eq(h.row(event).S("state")) + h.eq(field(h.statusOf(event), "phase")) + }) + }) + t.Run("empty listing", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_parent_whose_only_turn_was_lost_names_its_empty_listing", func(h *hl) { + event := h.queuedEvent(regOpts{}) + record := h.attemptOn(event, h.host, nil) + h.eq(str(record, "deliveryState")) + h.hostLoses(str(record, "turnId"), true) + h.eq(append([]any{}, turnIDs(h.host.threads[parent].turns)...)) + h.clock.Advance(5) + h.tick() + h.eq(field(h.statusOf(event), "phase")) + h.clock.Advance(120) + h.eq(field(h.tick().asDict(), "turnsUndecided")) + h.eq(field(h.statusOf(event), "phase")) + h.eq(field(h.completionDelivery(), "turnCheck")) + outcome := h.reconcile(str(record, "requestId"), h.host) + reading := sub(outcome, "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "undecided")}) + h.eq(field(sub(sub(outcome, "record"), "reconciliation"), "recipientTurnsChecked")) + h.eq(len(h.host.sends)) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("a found turn clears the name", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_an_undecided_reading_is_cleared_once_the_turn_is_found", func(h *hl) { + event, first, _ := h.dispatched() + h.exec("UPDATE attempts SET recipient_scan = ? WHERE request_id = ?", "turn_check_undecided:listing_bounded", first) + h.clock.Advance(120) + h.tick() + h.eq(field(h.statusOf(event), "phase")) + }) + }) + t.Run("a failed read keeps the name", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_an_undecided_name_survives_a_reconcile_whose_read_fails", func(h *hl) { + event, first, _ := h.dispatched() + h.exec("UPDATE attempts SET recipient_scan = ? WHERE request_id = ?", "turn_check_undecided:listing_bounded", first) + h.clock.Advance(120) + h.host.readFailures["find_dispatched_turn"] = true + h.eq(field(sub(h.reconcile(first, h.host), "recipientTurn"), "finding")) + h.eq(field(h.statusOf(event), "phase")) + h.eq(field(h.completionDelivery(), "turnCheck")) + }) + }) + t.Run("the daemon records an undecided reading once", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_the_daemon_records_an_undecided_reading_once", func(h *hl) { + _, _, turn := h.dispatched() + message := h.messageOf(turn) + h.hostLoses(turn, true) + h.host.startTurn(parent, "parent-later", "completed", "") + h.item(parent, "parent-later", message, "userMessage") + for n := 0; n < 201; n++ { + h.item(parent, "parent-later", fmt.Sprintf("work item %d", n), "userMessage") + } + h.clock.Advance(120) + first := h.tick().asDict() + h.eq([]any{field(first, "turnsUndecided"), field(first, "turnsLost")}) + h.clock.Advance(20) + h.eq(field(h.tick().asDict(), "turnsUndecided")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a message under an unfamiliar item type", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_message_the_host_types_unfamiliarly_is_named_and_not_sent_again", func(h *hl) { + event, first, turn := h.dispatched() + message := h.messageOf(turn) + h.hostReloadsLosing(turn) + h.item(parent, turn, message, "hookPrompt") + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(h.attemptsFor(event)[0].Opt("recipient_scan")) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "undecided")}) + h.eq(strings.Contains(str(reading, "detail"), "hookPrompt")) + }) + }) +} + +func Test21_HLT08_reconcile_names_the_loss_and_queues_the_redelivery_without_sending(t *testing.T) { + plain := func(name string, stage func(h *hl, turn string), full bool) { + t.Run(name, func(t *testing.T) { + mirror(t, hlt, name, func(h *hl) { + event, first, turn := h.dispatched() + stage(h, turn) + h.clock.Advance(120) + outcome := h.reconcile(first, h.host) + reading := sub(outcome, "recipientTurn") + if full { + h.eq([]any{field(reading, "turnId"), field(reading, "finding")}) + h.eq(field(sub(sub(outcome, "record"), "reconciliation"), "recipientTurnsChecked")) + h.eq(field(outcome, "state")) + h.eq(field(outcome, "redelivery")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + return + } + h.eq(field(reading, "finding")) + h.eq(h.row(event).S("state")) + }) + }) + } + plain("ReconcileReportsTheRecipientTurn.test_reconcile_names_the_loss_and_queues_the_redelivery_without_sending", func(h *hl, turn string) { h.hostLoses(turn, true) }, true) + plain("ReconcileReportsTheRecipientTurn.test_a_short_turn_after_the_send_is_read_through_and_the_loss_still_found", func(h *hl, turn string) { + h.hostLoses(turn, true) + h.host.startTurn(parent, "parent-later", "completed", "unrelated work") + }, false) + plain("ReconcileReportsTheRecipientTurn.test_the_items_of_a_turn_listed_before_the_send_end_the_scan", func(h *hl, turn string) { + t := h.host.threads[parent] + var earlier [][3]string + for n := 0; n < 250; n++ { + earlier = append(earlier, [3]string{"parent-earlier", fmt.Sprintf("earlier item %d", n), "userMessage"}) + } + t.items = append(earlier, t.items...) + h.hostLoses(turn, true) + }, false) +} + +func Test21_HLT09_an_unreadable_turn_list_is_unchecked_and_changes_nothing(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_an_unreadable_turn_list_is_reported_as_unchecked_and_changes_nothing", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.host.readFailures["find_dispatched_turn"] = true + outcome := h.reconcile(first, h.host) + h.eq(field(sub(outcome, "recipientTurn"), "finding")) + h.eq(field(sub(sub(outcome, "record"), "reconciliation"), "recipientTurnsChecked")) + h.eq(h.row(event).S("state")) + }) +} + +func Test21_HLT10_the_delivery_token_vetoes_a_missing_turn_row(t *testing.T) { + t.Run("token in the parent's items", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_the_delivery_token_in_the_parents_items_vetoes_a_missing_turn_row", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, false) + h.clock.Advance(120) + h.eq(field(sub(h.reconcile(first, h.host), "recipientTurn"), "finding")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + }) + }) + t.Run("token behind other dropped turns' items", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_token_behind_the_items_of_other_dropped_turns_still_vetoes_the_loss", func(h *hl) { + event, first, turn := h.dispatched() + h.clock.Advance(5) + h.host.startTurn(parent, "parent-later", "completed", "later work") + h.host.startTurn(parent, "parent-later-2", "completed", "more later work") + for _, dropped := range []string{turn, "parent-later", "parent-later-2"} { + h.hostLoses(dropped, false) + } + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + h.eq(h.journalled(HostLostTurn)) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status")}) + }) + }) + t.Run("a steered turn begun before the send", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_start_that_steered_a_turn_begun_before_the_send_is_present", func(h *hl) { + h.host.startTurn(parent, "parent-running", "inProgress", "") + h.clock.Advance(300) + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"steer_existing"} + record := h.attemptOn(event, h.host, nil) + h.eq(field(record, "turnId")) + h.clock.Advance(120) + h.eq(field(sub(h.reconcile(str(record, "requestId"), h.host), "recipientTurn"), "finding")) + h.eq(h.row(event).S("state")) + }) + }) +} + +func Test21_HLT11_the_lost_attempt_is_the_once_count(t *testing.T) { + t.Run("reconciling it again changes nothing", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_reconciling_a_lost_attempt_again_changes_nothing", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + mustDo(t, h.store.Transaction(h.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, h.store, "UPDATE attempts SET state = ? WHERE request_id = ?", HostLostTurn, first); err != nil { + return err + } + _, err := execSQL(ctx, h.store, "UPDATE deliveries SET state = ?, dispatch_evidence = ? WHERE event_id = ?", Queued, HostLostTurn, event) + return err + })) + second := h.attemptOn(event, h.host, nil) + h.eq(str(second, "deliveryState")) + h.eq(field(sub(h.reconcile(first, h.host), "recipientTurn"), "finding")) + h.eq(h.attemptStates(event)) + h.eq(h.journalled(HostLostTurn)) + h.hostLoses(str(second, "turnId"), true) + for i := 0; i < 3; i++ { + h.clock.Advance(120) + h.tick() + } + h.eq(h.row(event).Opt("hold_reason")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a reconcile racing the daemon's settlement", func(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_reconcile_racing_the_daemons_settlement_cannot_undo_the_loss", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + fired := 0 + racing := &hooked{Adapter: h.host} + racing.findDispatched = func(thread, turn string, sentAt float64) (TurnPresence, error) { + if fired == 0 { + fired++ + _, err := h.rc.CheckDispatchedTurn(h.ctx, first, h.host) + mustDo(t, err) + } + return h.host.FindDispatchedTurn(thread, turn, sentAt) + } + outcome := h.reconcile(first, racing) + h.eq(fired) + h.eq(field(outcome, "state")) + h.eq(field(sub(outcome, "recipientTurn"), "finding")) + h.eq(field(outcome, "redelivery")) + h.eq(h.attemptStates(event)) + h.eq(h.row(event).S("state")) + h.eq(h.journalled(HostLostTurn)) + second := h.attemptOn(event, h.host, nil) + h.hostLoses(str(second, "turnId"), true) + for i := 0; i < 3; i++ { + h.clock.Advance(120) + h.tick() + } + h.eq(h.row(event).Opt("hold_reason")) + h.eq(len(h.host.sends)) + }) + }) +} + +func Test21_HLT12_a_recorded_acknowledgement_wins_over_a_missing_turn(t *testing.T) { + mirror(t, hlt, "ReconcileReportsTheRecipientTurn.test_a_recorded_acknowledgement_wins_over_a_missing_turn", func(h *hl) { + event, first, turn := h.dispatched() + _, err := h.ack.Acknowledge(h.ctx, event, "ack-later", AckProof(event, "ack-later"), true, nil, nil) + mustDo(t, err) + h.hostLoses(turn, true) + h.clock.Advance(120) + outcome := h.reconcile(first, h.host) + h.eq(field(sub(outcome, "recipientTurn"), "finding")) + h.eq(field(outcome, "redelivery")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.tick() + h.eq(len(h.host.sends)) + }) +} + +func Test21_HLT13_the_controls_read_as_before(t *testing.T) { + t.Run("a lost acknowledgement keeps waiting", func(t *testing.T) { + mirror(t, hlt, "TheControlsReadAsBefore.test_a_lost_acknowledgement_keeps_waiting_and_is_read_as_present", func(h *hl) { + event, first, turn := h.dispatched() + h.host.finishTurn(parent, turn, "interrupted") + h.clock.Advance(120) + for i := 0; i < 2; i++ { + h.tick() + h.clock.Advance(120) + } + item := h.statusOf(event) + h.eq([]any{field(item, "state"), field(item, "reported"), field(item, "phase")}) + h.eq(len(h.host.sends)) + h.eq(h.journalled(HostLostTurn)) + outcome := h.reconcile(first, h.host) + reading := sub(outcome, "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status")}) + h.eq(field(sub(sub(outcome, "record"), "reconciliation"), "recipientTurnsChecked")) + }) + }) + t.Run("a normal acknowledgement settles before any check", func(t *testing.T) { + mirror(t, hlt, "TheControlsReadAsBefore.test_a_normal_acknowledgement_settles_before_any_check_is_owed", func(h *hl) { + event, _, _ := h.dispatched() + h.clock.Advance(5) + h.acknowledge(event, "ack-turn") + var lookups []string + counting := countingLookups(h.host, &lookups) + checks := &TurnChecks{Reconciler: h.rc} + for i := 0; i < 2; i++ { + h.clock.Advance(120) + h.tickWith(defaultTick(), counting, checks) + } + h.eq(h.row(event).S("state")) + h.eq(len(h.host.sends)) + h.eq(append([]any{}, anySlice(lookups)...)) + h.eq(h.journalled(HostLostTurn)) + }) + }) +} + +func anySlice(list []string) []any { + out := make([]any, len(list)) + for i, v := range list { + out[i] = v + } + return out +} + +func Test21_HLT14_assignment_show_names_the_next_actor_for_each_completion_state(t *testing.T) { + run := func(name string, body func(h *hl)) { + t.Run(name, func(t *testing.T) { + mirror(t, hlt, "AssignmentShowNamesTheNextActor."+name, func(h *hl) { + body(h) + h.eq(h.nextAction()) + }) + }) + } + run("test_a_delivered_completion_waits_for_the_parents_acknowledgement", func(h *hl) { h.dispatched() }) + run("test_a_completion_not_yet_sent_is_still_the_daemons", func(h *hl) { h.queuedEvent(regOpts{}) }) + run("test_an_acknowledged_completion_waits_for_the_parents_verification", func(h *hl) { + event, _, _ := h.dispatched() + h.clock.Advance(5) + h.acknowledge(event, "ack-turn") + }) + run("test_a_recorded_acknowledgement_is_the_daemons_to_verify", func(h *hl) { + event, _, _ := h.dispatched() + _, err := h.ack.Acknowledge(h.ctx, event, "ack-later", AckProof(event, "ack-later"), true, nil, nil) + mustDo(h.t, err) + }) + run("test_a_refused_acknowledgement_is_the_parents_to_restate", func(h *hl) { + event, _, _ := h.dispatched() + _, err := h.ack.Acknowledge(h.ctx, event, "ack-early", AckProof(event, "ack-early"), true, nil, nil) + mustDo(h.t, err) + h.exec("UPDATE ack_evidence SET last_reason = ? WHERE event_id = ?", "ack_turn_unverified", event) + }) + run("test_an_uncertain_send_is_the_daemons_to_reconcile", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"transport_unknown"} + h.attemptOn(event, h.host, nil) + h.eq(h.row(event).S("state")) + }) + run("test_an_interrupted_claim_is_the_daemons_to_reconcile", func(h *hl) { + event := h.queuedEvent(regOpts{}) + h.exec("UPDATE deliveries SET state = 'sending' WHERE event_id = ?", event) + }) + run("test_an_inbox_only_completion_is_the_parents_to_acknowledge", func(h *hl) { + event, _, _ := h.dispatched() + h.exec("UPDATE deliveries SET state = 'inbox_only', hold_reason = 'push_channel_closed' WHERE event_id = ?", event) + }) + t.Run("test_a_claimed_completion_without_an_ack_is_the_parents_to_acknowledge", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_claimed_completion_without_an_ack_is_the_parents_to_acknowledge", func(h *hl) { + event, _, _ := h.dispatched() + _, err := h.ack.ClaimVerification(h.ctx, event, "parent-reading") + mustDo(t, err) + h.eq(h.nextAction()) + }) + }) +} + +func Test21_HLT15_the_host_loss_is_named_at_every_step_of_its_recovery(t *testing.T) { + t.Run("every step", func(t *testing.T) { + mirror(t, hlt, "AssignmentShowNamesTheNextActor.test_the_host_loss_is_named_at_every_step_of_its_recovery", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.eq(h.nextAction()) + h.eq(field(h.completionDelivery(), "hostLostAttempts")) + h.exec("UPDATE deliveries SET state = 'sending' WHERE event_id = ?", event) + h.eq(h.nextAction()) + h.exec("UPDATE deliveries SET state = 'queued' WHERE event_id = ?", event) + record := h.attemptOn(event, h.host, nil) + h.eq(str(record, "deliveryState")) + h.eq(h.nextAction()) + h.eq(field(h.completionDelivery(), "hostLostAttempts")) + h.hostLoses(str(record, "turnId"), true) + h.clock.Advance(120) + h.reconcile(str(record, "requestId"), h.host) + h.eq(h.row(event).Opt("hold_reason")) + h.eq(h.nextAction()) + h.eq(field(h.completionDelivery(), "hostLostAttempts")) + }) + }) + t.Run("held for another reason after a loss", func(t *testing.T) { + mirror(t, hlt, "AssignmentShowNamesTheNextActor.test_a_redelivery_held_for_another_reason_is_still_the_parents_after_a_loss", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.exec("UPDATE deliveries SET state = 'withheld_pre_send', hold_reason = 'attempt_cap' WHERE event_id = ?", event) + h.eq(h.nextAction()) + }) + }) + t.Run("an uncertain redelivery after a loss", func(t *testing.T) { + mirror(t, hlt, "AssignmentShowNamesTheNextActor.test_an_uncertain_redelivery_after_a_loss_is_reconciled_not_resent", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.host.script = []string{"transport_unknown"} + h.attemptOn(event, h.host, nil) + h.eq(h.row(event).S("state")) + h.eq(h.nextAction()) + }) + }) + t.Run("a corrected completion reads its own delivery", func(t *testing.T) { + mirror(t, hlt, "AssignmentShowNamesTheNextActor.test_a_corrected_completion_reads_its_own_delivery", func(h *hl) { + first := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + h.attemptOn(first, h.host, nil) + h.clock.Advance(5) + h.acknowledge(first, "ack-turn") + _, err := h.ack.RecordVerdict(h.ctx, first, "needs_changes", "v1", nil, []any{Obj{{Key: "id", Value: "c1"}, {Key: "verdict", Value: "needs_changes"}, {Key: "note", Value: "fix the shape"}}}, nil, nil) + mustDo(t, err) + correction := h.one("SELECT event_id FROM deliveries WHERE kind = ?", Revision).S("event_id") + h.attemptOn(correction, h.host, nil) + h.clock.Advance(300) + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(t, err) + _, err = BindAnchor(h.ctx, h.store, h.clock, h.rid, r.Generation, "revision-turn") + mustDo(t, err) + payload := h.readyPayload(h.rid, r.Generation, []string{h.artifact("revised.txt", "the corrected deliverable")}, 1, turnRef{child, "revision-turn", "completed"}) + _, err = h.accept(payload, storeAcceptNone) + mustDo(t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(t, err) + h.eq(field(h.assignment(), "state")) + h.eq(h.nextAction()) + record := h.attemptOn(str(payload, "eventId"), h.host, nil) + h.eq(str(record, "deliveryState")) + h.eq(h.nextAction()) + h.hostLoses(str(record, "turnId"), true) + h.clock.Advance(120) + h.reconcile(str(record, "requestId"), h.host) + h.eq(h.nextAction()) + }) + }) +} + +func Test21_HLT16_only_the_delivered_user_message_vetoes_a_loss(t *testing.T) { + t.Run("a message only under another turn", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_message_found_only_under_another_turn_is_read_again_and_its_loss_caught", func(h *hl) { + event, first, turn := h.dispatched() + message := h.messageOf(turn) + h.hostReloadsLosing(turn) + h.host.startTurn(parent, "parent-later", "completed", "") + h.item(parent, "parent-later", message, "userMessage") + h.clock.Advance(120) + h.tick() + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status"), field(reading, "undecided")}) + h.eq(strings.Contains(str(reading, "detail"), "parent-later")) + h.eq(len(h.host.sends)) + h.hostReloadsLosing("parent-later") + h.clock.Advance(20) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a relay command echoing the request id", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_relay_command_echoing_the_request_id_does_not_veto_the_loss", func(h *hl) { + event, first, turn := h.dispatched() + h.hostReloadsLosing(turn) + h.host.startTurn(parent, "parent-later", "completed", "") + h.echo("parent-later", dumps(Obj{{Key: "delivery", Value: Obj{{Key: "requestId", Value: first}}}})) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + }) + }) + t.Run("a file the parent wrote", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_file_the_parent_wrote_with_the_request_id_does_not_veto_the_loss", func(h *hl) { + event, first, turn := h.dispatched() + h.hostReloadsLosing(turn) + h.host.startTurn(parent, "parent-later", "completed", "") + h.item(parent, "parent-later", "+ waiting on "+first, "fileChange") + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + }) + }) +} + +func Test21_HLT17_listed_turns_are_read_from_their_own_items(t *testing.T) { + t.Run("a long finished turn is read from its first item", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_a_long_finished_turn_is_read_from_its_first_item", func(h *hl) { + _, first, turn := h.dispatched() + for n := 0; n < 300; n++ { + h.item(parent, turn, fmt.Sprintf("work item %d", n), "commandExecution") + } + h.host.finishTurn(parent, turn, "completed") + h.clock.Advance(120) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status"), field(reading, "undecided")}) + }) + }) + t.Run("an in-progress listed turn is present unread", func(t *testing.T) { + mirror(t, hlt, "TheHostListsALostTurnAfterAReload.test_an_in_progress_listed_turn_is_present_without_reading_its_items", func(h *hl) { + _, first, turn := h.dispatched() + h.dropItems(turn) + h.clock.Advance(120) + reading := sub(h.reconcile(first, h.host), "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status")}) + }) + }) +} + +func turnIDs(turns []TurnInfo) []any { + out := make([]any, len(turns)) + for i, t := range turns { + out[i] = t.TurnID + } + return out +} diff --git a/internal/relay/delivery/hostloss_c_test.go b/internal/relay/delivery/hostloss_c_test.go new file mode 100644 index 00000000..a8cbb611 --- /dev/null +++ b/internal/relay/delivery/hostloss_c_test.go @@ -0,0 +1,606 @@ +package delivery + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_host_lost_turn.py HLT-18..HLT-27 (see hostloss_a_test.go for the method). + +func Test21_HLT18_an_ack_before_the_send_is_confirmed_is_kept_and_completed(t *testing.T) { + t.Run("an ACK in the delivery turn confirms the send", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_in_the_delivery_turn_confirms_the_send_and_the_verdict_follows", func(h *hl) { + event, _, turn := h.uncertainDelivery(true) + claim, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(t, err) + h.eq(claim) + h.eq(field(h.cliAck(event, turn, nil), "_verified")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.eq(h.evidenceOf(event)) + h.eq(field(h.cliVerdict(event, turn), "verdict")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("an unconfirmable ACK is kept", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_the_relay_cannot_confirm_yet_is_kept_not_refused", func(h *hl) { + event, _, turn := h.uncertainDelivery(false) + _, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(t, err) + h.eq(field(h.cliAck(event, turn, nil), "_verified")) + row := h.ackRow(event) + h.eq([]any{row.Opt("verified"), row.Opt("last_reason")}) + h.eq(h.row(event).S("state")) + h.eq(h.nextAction()) + }) + }) + t.Run("a kept ACK completes once the delivery is confirmed", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_kept_ack_completes_once_the_delivery_is_confirmed", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + _, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(t, err) + h.cliAck(event, turn, nil) + h.clock.Advance(1) + h.tick() + h.eq(h.row(event).S("state")) + h.item(parent, turn, "requestId: "+request, "") + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + h.eq(h.nextAction()) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a confirmed delivery with a kept ACK", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_confirmed_delivery_with_a_kept_ack_is_the_daemons_to_verify", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + _, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(t, err) + h.cliAck(event, turn, nil) + h.item(parent, turn, "requestId: "+request, "") + h.reconcile(request, h.host) + h.eq(h.row(event).S("state")) + h.eq(h.nextAction()) + }) + }) + t.Run("a verdict completes a kept ACK first", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_verdict_completes_a_kept_ack_first", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + _, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(t, err) + h.cliAck(event, turn, nil) + h.item(parent, turn, "requestId: "+request, "") + h.eq(field(h.cliVerdict(event, turn), "verdict")) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("the daemon confirms a kept ACK through its turn", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_kept_ack_is_confirmed_by_the_daemon_through_its_turn", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + h.cliAck(event, turn, nil) + thread := h.host.threads[parent] + position := 0 + for i, item := range thread.items { + if item[0] == turn { + position = i + break + } + } + thread.items = append(thread.items[:position], append([][3]string{{turn, "requestId: " + request, "userMessage"}}, thread.items[position:]...)...) + for n := 0; n < 250; n++ { + h.item(parent, turn, fmt.Sprintf("work item %d", n), "commandExecution") + } + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + }) + }) + t.Run("verify-acks confirms a kept ACK's delivery", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_manual_verify_acks_confirms_a_kept_acks_delivery_through_its_turn", func(h *hl) { + event, _ := h.foldedDelivery(true, 10, 250) + _, err := h.ack.Acknowledge(h.ctx, event, "folded", AckProof(event, "folded"), true, nil, nil) + mustDo(t, err) + _, err = VerifyAcksCommand(h.ctx, h.ack, h.rc, h.host, 8) + mustDo(t, err) + h.eq(h.row(event).S("state")) + }) + }) +} + +func Test21_HLT19_an_ack_while_the_relay_is_still_sending_is_kept(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_while_the_relay_is_still_sending_is_kept_and_completes_after", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + var reads []any + counting := countingReads(h.host, &reads) + var keptState string + var keptRecord Obj + during := &hooked{Adapter: h.host} + during.send = func(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + receipt, err := h.host.SendMessage(requestID, thread, message, settings) + keptState = h.row(event).S("state") + keptRecord = h.cliAck(event, str(receipt, "turnId"), counting) + return receipt, err + } + h.attemptOn(event, during, nil) + h.eq(keptState) + h.eq(field(keptRecord, "_verified")) + h.eq(append([]any{}, reads...)) + h.eq(h.row(event).S("state")) + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + }) +} + +func Test21_HLT20_host_reads_only_when_needed(t *testing.T) { + t.Run("an ordinary ACK", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ordinary_ack_makes_no_confirmation_reads", func(h *hl) { + event, _, _ := h.dispatched() + h.clock.Advance(5) + turn := h.host.startTurn(parent, "ack-turn", "inProgress", "") + var reads []any + h.eq(field(h.cliAck(event, turn.TurnID, countingReads(h.host, &reads)), "_verified")) + h.eq(append([]any{}, reads...)) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("a wrong proof", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_wrong_proof_makes_no_host_reads", func(h *hl) { + event, _, turn := h.uncertainDelivery(true) + var reads []any + counting := countingReads(h.host, &reads) + _, err := AckCommand(h.ctx, h.ack, h.rc, counting, event, turn, "0000000000000000000000000000000000000000000000000000000000000000", nil) + requireReason(t, err, AckProofMismatch) + h.eq(append([]any{}, reads...)) + h.eq(h.row(event).S("state")) + }) + }) +} + +func Test21_HLT21_the_message_is_found_through_the_ack_turn_and_an_echo_confirms_nothing(t *testing.T) { + t.Run("deep in a long delivery turn", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_message_deep_in_a_long_delivery_turn_is_confirmed_through_the_ack_turn", func(h *hl) { + event, request, turn := h.uncertainDelivery(true) + for n := 0; n < 250; n++ { + h.item(parent, turn, fmt.Sprintf("work item %d", n), "commandExecution") + } + scan, err := h.host.FindToken(parent, request, 200, false) + mustDo(t, err) + h.eq(scan.Found) + h.eq(field(h.cliAck(event, turn, nil), "_verified")) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("deep in a folded turn", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_a_message_deep_in_a_folded_turn_is_found_through_the_ack_turn", func(h *hl) { + event, request := h.foldedDelivery(true, 10, 250) + h.eq(field(h.cliAck(event, "folded", nil), "_verified")) + h.eq(h.row(event).S("state")) + h.host.finishTurn(parent, "folded", "completed") + h.clock.Advance(120) + outcome, err := h.rc.CheckDispatchedTurn(h.ctx, request, h.host) + mustDo(t, err) + reading := sub(outcome, "recipientTurn") + h.eq([]any{field(reading, "finding"), field(reading, "status"), field(reading, "undecided")}) + }) + }) + t.Run("an echo in the ACK turn", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_echo_in_the_ack_turn_does_not_confirm_the_send", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + h.echo(turn, dumps(Obj{{Key: "delivery", Value: Obj{{Key: "requestId", Value: request}}}})) + h.eq(field(h.cliAck(event, turn, nil), "_verified")) + h.eq(h.row(event).S("state")) + h.eq(h.evidenceOf(event)) + }) + }) + t.Run("an echo alone", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_echo_alone_never_confirms_an_uncertain_send", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + h.echo(turn, "status: attempt "+request+" held_uncertain") + h.eq(field(h.reconcile(request, h.host), "state")) + h.eq(h.row(event).S("state")) + }) + }) +} + +func Test21_HLT22_an_ack_from_the_turn_the_send_was_folded_into(t *testing.T) { + t.Run("verified", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_from_the_turn_the_send_was_folded_into_is_verified", func(h *hl) { + event, _ := h.foldedDelivery(true, 0, 0) + h.eq(field(h.cliAck(event, "folded", nil), "_verified")) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("dispatch_turn_id")}) + h.eq(len(h.host.sends)) + }) + }) + t.Run("kept until the send is confirmed", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_from_a_folded_turn_is_kept_until_the_send_is_confirmed", func(h *hl) { + event, request := h.foldedDelivery(false, 0, 0) + h.eq(field(h.cliAck(event, "folded", nil), "_verified")) + h.eq(h.ackRow(event).Opt("last_reason")) + h.item(parent, "folded", "requestId: "+request, "") + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + h.eq(len(h.host.sends)) + }) + }) +} + +func Test21_HLT23_an_ack_answers_only_attempts_sent_before_it_was_authored(t *testing.T) { + t.Run("a kept ACK does not hide the next attempt's loss", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_kept_for_an_attempt_that_never_sent_does_not_hide_the_next_attempts_loss", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + first := str(h.attemptOn(event, h.host, nil), "requestId") + h.clock.Advance(2) + h.host.startTurn(parent, "ack-turn", "completed", "") + h.eq(field(h.cliAck(event, "ack-turn", nil), "_verified")) + h.host.ledger[first] = preSendRejection(first) + h.reconcile(first, h.host) + h.clock.Advance(100000) + second := h.attemptOn(event, h.host, at(h.clock.Now())) + h.eq(str(second, "deliveryState")) + h.hostReloadsLosing(str(second, "turnId")) + h.clock.Advance(120) + h.tick() + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a kept ACK is not promoted for a later folded attempt", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_a_kept_ack_is_not_promoted_for_a_later_attempt_folded_into_the_same_turn", func(h *hl) { + event, first := h.foldedDelivery(false, 0, 0) + _, err := h.ack.Acknowledge(h.ctx, event, "folded", AckProof(event, "folded"), true, nil, nil) + mustDo(t, err) + h.host.ledger[first] = preSendRejection(first) + h.reconcile(first, h.host) + h.clock.Advance(100000) + h.host.script = []string{"in_progress"} + second := str(h.attemptOn(event, h.host, at(h.clock.Now())), "requestId") + h.item(parent, "folded", "requestId: "+second, "") + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + row := h.ackRow(event) + h.eq([]any{row.Opt("verified"), row.Opt("last_reason")}) + h.eq(h.nextAction()) + h.eq(field(h.cliAck(event, "folded", nil), "_verified")) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("an ACK authored after the send keeps the delivery out of the pass", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementBeforeTheSendIsConfirmed.test_an_ack_authored_after_the_send_keeps_the_delivery_out_of_the_turn_pass", func(h *hl) { + event, _, turn := h.dispatched() + h.clock.Advance(5) + _, err := h.ack.Acknowledge(h.ctx, event, "ack-later", AckProof(event, "ack-later"), true, nil, nil) + mustDo(t, err) + var reads []string + counting := countingLookups(h.host, &reads) + h.hostLoses(turn, true) + h.clock.Advance(120) + h.adapter = counting + h.tick() + h.eq(anySlice(reads)) + h.eq(h.attemptStates(event)) + h.eq(len(h.host.sends)) + }) + }) +} + +func (h *hl) confirmsDuringTheScan(event, turn string) *hooked { + fired := false + w := &hooked{Adapter: h.host} + w.findToken = func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + scan, err := h.host.FindToken(thread, token, limit, messageOnly) + if !fired { + fired = true + _, cerr := h.rc.ConfirmDelivery(h.ctx, event, h.host, turn) + mustDo(h.t, cerr) + } + return scan, err + } + return w +} + +func Test21_HLT24_every_settlement_is_a_compare_and_set(t *testing.T) { + t.Run("a replacement ACK during the read", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_replacement_ack_written_during_the_read_is_not_promoted_with_the_old_evidence", func(h *hl) { + event, request, turn := h.uncertainDelivery(false) + h.cliAck(event, turn, nil) + h.item(parent, turn, "requestId: "+request, "") + h.reconcile(request, h.host) + h.eq(h.row(event).S("state")) + later := h.host.startTurn(parent, "later-turn", "inProgress", "") + fired := false + replaces := &hooked{Adapter: h.host} + replaces.readTurn = func(thread, turn string) (*TurnInfo, error) { + if !fired { + fired = true + _, err := h.ack.Acknowledge(h.ctx, event, later.TurnID, AckProof(event, later.TurnID), true, nil, nil) + mustDo(t, err) + } + return h.host.ReadTurn(thread, turn) + } + results, err := h.ack.VerifyPendingAcks(h.ctx, replaces, 8, nil) + mustDo(t, err) + h.eq(outcomes(results)) + row := h.one("SELECT ack_turn_id, verified FROM acks WHERE event_id = ?", event) + h.eq([]any{row.Opt("ack_turn_id"), row.Opt("verified")}) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("a reader that found nothing cannot undo a confirmation", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_reader_that_found_nothing_cannot_undo_a_confirmation_made_meanwhile", func(h *hl) { + event, request, turn := h.uncertainDelivery(true) + for n := 0; n < 250; n++ { + h.item(parent, turn, fmt.Sprintf("work item %d", n), "commandExecution") + } + outcome := h.reconcile(request, h.confirmsDuringTheScan(event, turn)) + h.eq(field(outcome, "evidence")) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("a reader that found nothing cannot undo a pre-send rejection", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_reader_that_found_nothing_cannot_undo_a_pre_send_rejection_made_meanwhile", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + fired := false + rejects := &hooked{Adapter: h.host} + rejects.findToken = func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + scan, err := h.host.FindToken(thread, token, limit, messageOnly) + if !fired { + fired = true + h.host.ledger[request] = preSendRejection(request) + h.reconcile(request, h.host) + } + return scan, err + } + outcome := h.reconcile(request, rejects) + h.eq(truthy(field(outcome, "changed"))) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + h.clock.Advance(100000) + h.eq(field(h.attemptOn(event, h.host, at(h.clock.Now())), "attemptNo")) + }) + }) + t.Run("a refused stale write leaves the attempt owed", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_refused_stale_write_leaves_the_attempt_owed_to_the_next_tick", func(h *hl) { + event, request, turn := h.uncertainDelivery(true) + for n := 0; n < 250; n++ { + h.item(parent, turn, fmt.Sprintf("work item %d", n), "commandExecution") + } + policy := defaultTick() + policy.maxSendsTick = 0 + h.tickWith(policy, h.confirmsDuringTheScan(event, turn), h.checks) + h.eq(h.one("SELECT retry_required FROM reconcile_gate WHERE request_id = ?", request).Opt("retry_required")) + h.eq(h.evidenceOf(event)) + }) + }) + t.Run("a later unreadable receipt keeps a pre-send rejection", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_later_reconcile_that_cannot_read_the_receipt_keeps_a_pre_send_rejection", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.host.ledger[request] = preSendRejection(request) + h.reconcile(request, h.host) + h.eq(h.evidenceOf(event)) + delete(h.host.ledger, request) + h.reconcile(request, h.host) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + }) + }) +} + +func outcomes(results []any) []any { + out := []any{} + for _, r := range results { + out = append(out, field(r.(Obj), "outcome")) + } + return out +} + +func Test21_HLT25_the_sender_and_a_concurrent_reconcile_never_undo_each_other(t *testing.T) { + t.Run("a reconcile during the send that found nothing", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_reconcile_during_the_send_that_found_nothing_leaves_the_accepted_send_to_the_next_tick", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + var seen Obj + w := &hooked{Adapter: h.host} + w.send = func(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + seen = h.reconcile(requestID, h.host) + return h.host.SendMessage(requestID, thread, message, settings) + } + result := h.attemptOn(event, w, nil) + h.eq(field(seen, "state")) + h.eq(truthy(field(result, "_settledElsewhere"))) + h.eq(h.row(event).S("state")) + h.clock.Advance(20) + policy := defaultTick() + policy.maxSendsTick = 0 + h.tickWith(policy, h.host, h.checks) + h.eq(h.row(event).S("state")) + h.eq(len(h.host.sends)) + }) + }) + t.Run("a reconcile during the send that confirmed it", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_reconcile_during_the_send_that_confirmed_it_survives_the_senders_unknown_result", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + w := &hooked{Adapter: h.host} + w.send = func(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + receipt, err := h.host.SendMessage(requestID, thread, message, settings) + h.host.startTurn(parent, "", "inProgress", "requestId: "+requestID) + h.reconcile(requestID, h.host) + return receipt, err + } + result := h.attemptOn(event, w, nil) + h.eq(truthy(field(result, "_settledElsewhere"))) + h.eq(field(result, "_deliveryState")) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("dispatch_turn_id")}) + h.eq(h.evidenceOf(event)) + }) + }) + t.Run("a reconcile in another process that read the send in flight", func(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_reconcile_that_read_the_send_in_flight_cannot_undo_the_senders_settlement", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + readDone, settled := make(chan struct{}), make(chan struct{}) + var once sync.Once + var seen Obj + var failure error + var wg sync.WaitGroup + reconcileElsewhere := func(requestID string) { + defer wg.Done() + other, err := store.Open(context.Background(), h.store.Path, "") + if err != nil { + failure = err + once.Do(func() { close(readDone) }) + return + } + defer func() { _ = other.Close() }() + waits := &hooked{Adapter: h.host} + waits.findToken = func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + scan, err := h.host.FindToken(thread, token, limit, messageOnly) + once.Do(func() { close(readDone) }) + <-settled + return scan, err + } + seen, failure = NewReconciler(NewService(other, h.clock)).ReconcileAttempt(context.Background(), requestID, waits, nil) + } + w := &hooked{Adapter: h.host} + w.send = func(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + wg.Add(1) + go reconcileElsewhere(requestID) + <-readDone + return h.host.SendMessage(requestID, thread, message, settings) + } + result := h.attemptOn(event, w, nil) + close(settled) + wg.Wait() + h.eq(anyErrors(failure)) + h.eq(str(result, "deliveryState")) + h.eq(truthy(field(seen, "changed"))) + h.eq(h.attemptStates(event)) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("dispatch_turn_id")}) + }) + }) +} + +func anyErrors(err error) []any { + if err == nil { + return []any{} + } + return []any{err.Error()} +} + +func Test21_HLT26_a_typed_item_changes_the_fingerprint_and_readback(t *testing.T) { + mirror(t, hlt, "EverySettlementIsACompareAndSet.test_a_typed_item_leaves_the_fingerprint_and_readback_working", func(h *hl) { + h.item(parent, "t-x", "plain", "") + _, err := h.host.RecipientFingerprint(parent) + mustDo(t, err) + h.item(parent, "t-x", "del-echo-a1", "commandExecution") + after, err := h.host.RecipientFingerprint(parent) + mustDo(t, err) + h.eq(after) + anyItem, err := h.host.FindToken(parent, "del-echo-a1", 200, false) + mustDo(t, err) + h.eq(anyItem.Found) + only, err := h.host.FindToken(parent, "del-echo-a1", 200, true) + mustDo(t, err) + h.eq(only.Found) + }) +} + +func Test21_HLT27_an_ack_is_judged_against_the_delivery_it_read(t *testing.T) { + loseFirstTurnAndRequeue := func(h *hl) (string, string) { + event, _, lost := h.dispatched() + h.hostReloadsLosing(lost) + h.clock.Advance(120) + policy := defaultTick() + policy.maxSendsTick = 0 + h.tickWith(policy, h.host, h.checks) + h.eq(h.attemptStates(event)) + h.clock.Advance(10) + return event, lost + } + t.Run("a folded send confirmed during the read", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_a_folded_send_confirmed_during_the_read_keeps_the_ack_and_completes_it", func(h *hl) { + event, request := h.foldedDelivery(true, 0, 0) + record, err := h.ack.Acknowledge(h.ctx, event, "folded", AckProof(event, "folded"), true, nil, actsDuringTheTurnRead(h.host, func() { h.reconcile(request, h.host) })) + mustDo(t, err) + h.eq(field(record, "_verified")) + h.eq(h.row(event).S("state")) + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + }) + }) + t.Run("an ACK naming a lost turn", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_an_ack_naming_a_lost_turn_cannot_close_the_redelivery_confirmed_during_its_read", func(h *hl) { + event, lost := loseFirstTurnAndRequeue(h) + var acked Obj + w := &hooked{Adapter: h.host} + w.send = func(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + receipt, err := h.host.SendMessage(requestID, thread, message, settings) + var aerr error + acked, aerr = h.ack.Acknowledge(h.ctx, event, lost, AckProof(event, lost), true, nil, actsDuringTheTurnRead(h.host, func() { h.reconcile(requestID, h.host) })) + mustDo(t, aerr) + return receipt, err + } + h.attemptOn(event, w, nil) + h.eq(field(acked, "_verified")) + h.eq(h.row(event).S("state")) + h.clock.Advance(5) + h.tick() + h.eq(h.row(event).S("state")) + h.eq(h.ackRow(event).Opt("last_reason")) + h.eq(h.nextAction()) + }) + }) + t.Run("an ACK is not promoted onto an attempt sent during its read", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_an_ack_is_not_promoted_onto_an_attempt_sent_during_its_read", func(h *hl) { + event, first, lost := h.dispatched() + h.hostReloadsLosing(lost) + h.clock.Advance(120) + record, err := h.ack.Acknowledge(h.ctx, event, lost, AckProof(event, lost), true, nil, actsDuringTheTurnRead(h.host, func() { + _, err := h.rc.CheckDispatchedTurn(h.ctx, first, h.host) + mustDo(t, err) + h.attemptOn(event, h.host, nil) + })) + mustDo(t, err) + h.eq(h.attemptStates(event)) + h.eq(field(record, "_verified")) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("a pending ACK read against a delivery that moved", func(t *testing.T) { + mirror(t, hlt, "AnAcknowledgementIsJudgedAgainstTheDeliveryItRead.test_a_pending_ack_read_against_a_delivery_that_moved_is_checked_again", func(h *hl) { + event, request := h.foldedDelivery(true, 0, 0) + _, err := h.ack.Acknowledge(h.ctx, event, "folded", AckProof(event, "folded"), true, nil, nil) + mustDo(t, err) + results, err := h.ack.VerifyPendingAcks(h.ctx, actsDuringTheTurnRead(h.host, func() { h.reconcile(request, h.host) }), 8, nil) + mustDo(t, err) + h.eq(outcomes(results)) + h.eq(h.row(event).S("state")) + again, err := h.ack.VerifyPendingAcks(h.ctx, h.host, 8, nil) + mustDo(t, err) + h.eq(outcomes(again)) + h.eq(h.row(event).S("state")) + }) + }) +} diff --git a/internal/relay/delivery/hostloss_d_test.go b/internal/relay/delivery/hostloss_d_test.go new file mode 100644 index 00000000..56d06b41 --- /dev/null +++ b/internal/relay/delivery/hostloss_d_test.go @@ -0,0 +1,250 @@ +package delivery + +import ( + "errors" + "fmt" + "testing" +) + +// test_host_lost_turn.py HLT-28, HLT-29: the bridge adapter's recipient reads (BridgeReads), +// driven through the same call(method, params) pages the Python tests hand BridgeHostAdapter. + +const sentAtHLT = 1_700_000_000.0 + +func turnEntry(id any, startedAt any, status string) Obj { + return Obj{{Key: "id", Value: id}, {Key: "status", Value: status}, {Key: "startedAt", Value: startedAt}, {Key: "items", Value: []any{}}} +} + +func at1(offset float64) any { return sentAtHLT + offset } + +// pagesCall is _pages: a thread/turns/list answer per call, newest first, chained by cursor. +func pagesCall(t *testing.T, pages ...[]any) (func(string, Obj) (Obj, error), *[]Obj) { + var calls []Obj + return func(method string, params Obj) (Obj, error) { + if method != "thread/turns/list" { + t.Fatalf("unexpected call %s", method) + } + calls = append(calls, append(Obj(nil), params...)) + index := 0 + if c, ok := get(params, "cursor"); ok && c != nil { + fmt.Sscan(c.(string), &index) + } + var next any + if index+1 < len(pages) { + next = fmt.Sprint(index + 1) + } + data := []any{} + if index < len(pages) { + data = pages[index] + } + return Obj{{Key: "data", Value: data}, {Key: "nextCursor", Value: next}}, nil + }, &calls +} + +func (h *hl) lookup(call func(string, Obj) (Obj, error)) (TurnPresence, error) { + return BridgeReads{Call: call, Page: 2}.FindDispatchedTurn("thread", "wanted", sentAtHLT) +} + +func presenceTurn(p TurnPresence) []any { + return []any{p.Finding, p.Turn.TurnID, p.Turn.Status} +} + +func strs(list []string) []any { + out := []any{} + for _, v := range list { + out = append(out, v) + } + return out +} + +func Test21_HLT28_the_bridge_adapter_looks_back_only_to_the_send(t *testing.T) { + const cls = "TheAdapterLooksBackOnlyToTheSend." + t.Run("a turn on a later page", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_turn_on_a_later_page_is_found", func(h *hl) { + call, _ := pagesCall(t, []any{turnEntry("newer-1", at1(50), "completed"), turnEntry("newer-2", at1(40), "completed")}, []any{turnEntry("wanted", at1(1), "inProgress")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq(presenceTurn(p)) + }) + }) + t.Run("an older turn ends the search", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_turn_older_than_the_send_ends_the_search_without_reading_further", func(h *hl) { + call, calls := pagesCall(t, []any{turnEntry("newer", at1(30), "completed"), turnEntry("before", at1(-600), "completed")}, []any{turnEntry("wanted", at1(-900), "completed")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq([]any{p.Finding, p.Stop, p.Scanned}) + h.eq(len(*calls)) + }) + }) + t.Run("the end of the listing", func(t *testing.T) { + mirror(t, hlt, cls+"test_the_end_of_the_listing_is_absence", func(h *hl) { + call, _ := pagesCall(t, []any{turnEntry("newer", at1(30), "completed")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq([]any{p.Finding, p.Stop}) + }) + }) + t.Run("an empty listing", func(t *testing.T) { + mirror(t, hlt, cls+"test_an_empty_listing_is_not_evidence", func(h *hl) { + call, _ := pagesCall(t, []any{}) + _, err := h.lookup(call) + var empty *ListingEmpty + if !errors.As(err, &empty) { + t.Fatalf("want HostUnavailable (ListingEmpty), got %v", err) + } + }) + }) + t.Run("a bounded scan that never reached the send", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_bounded_scan_that_never_reached_the_send_is_not_evidence", func(h *hl) { + var pages [][]any + for n := 0; n < 25; n++ { + pages = append(pages, []any{turnEntry(fmt.Sprintf("newer-%d-0", n), at1(float64(1000-n)), "completed"), turnEntry(fmt.Sprintf("newer-%d-1", n), at1(float64(1000-n)), "completed")}) + } + call, calls := pagesCall(t, pages...) + _, err := h.lookup(call) + var bounded *ListingBounded + if !errors.As(err, &bounded) { + t.Fatalf("want HostUnavailable (ListingBounded), got %v", err) + } + h.eq(len(*calls)) + }) + }) + t.Run("250 later turns are read through", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_long_history_after_the_send_is_read_through_to_the_send", func(h *hl) { + var pages [][]any + for n := 0; n < 5; n++ { + var page []any + for m := 0; m < 50; m++ { + page = append(page, turnEntry(fmt.Sprintf("newer-%d-%d", n, m), at1(float64(1000-n)), "completed")) + } + pages = append(pages, page) + } + pages = append(pages, []any{turnEntry("before", at1(-600), "completed")}) + call, calls := pagesCall(t, pages...) + p, err := BridgeReads{Call: call, Page: 50}.FindDispatchedTurn("thread", "wanted", sentAtHLT) + mustDo(t, err) + h.eq([]any{p.Finding, p.Stop, p.Scanned}) + h.eq(len(*calls)) + }) + }) + t.Run("a steered turn begun before the send", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_steered_turn_begun_before_the_send_is_matched_before_the_cutoff", func(h *hl) { + call, _ := pagesCall(t, []any{turnEntry("wanted", at1(-600), "inProgress"), turnEntry("before", at1(-900), "completed")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq(p.Finding) + }) + }) + t.Run("an undated turn never ends the search", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_turn_with_no_start_time_never_ends_the_search", func(h *hl) { + call, _ := pagesCall(t, []any{turnEntry("undated", nil, "completed"), turnEntry("wanted", at1(2), "completed")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq(p.Finding) + }) + }) + t.Run("the request shape", func(t *testing.T) { + mirror(t, hlt, cls+"test_it_asks_for_the_newest_turns_first_without_their_items", func(h *hl) { + call, calls := pagesCall(t, []any{turnEntry("wanted", at1(1), "completed")}) + _, err := h.lookup(call) + mustDo(t, err) + h.eq((*calls)[0]) + }) + }) + t.Run("a match reads on to the first older turn", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_match_reads_on_to_the_first_turn_older_than_the_send", func(h *hl) { + call, calls := pagesCall(t, []any{turnEntry("newer", at1(30), "completed"), turnEntry("wanted", at1(1), "completed")}, + []any{turnEntry("between", at1(-10), "completed"), turnEntry("before", at1(-600), "completed")}, []any{turnEntry("older", at1(-900), "completed")}) + p, err := h.lookup(call) + mustDo(t, err) + h.eq([]any{p.Finding, p.Turn.TurnID, p.Scanned}) + h.eq(strs(p.Older)) + h.eq(len(*calls)) + }) + }) + t.Run("a failure reading on keeps the match", func(t *testing.T) { + mirror(t, hlt, cls+"test_a_failure_reading_on_after_a_match_keeps_the_match", func(h *hl) { + first := []any{turnEntry("newer", at1(30), "completed"), turnEntry("wanted", at1(1), "completed")} + call := func(method string, params Obj) (Obj, error) { + if c, _ := get(params, "cursor"); truthy(c) { + return nil, &HostError{Kind: "HostUnavailable", Message: "the next page could not be read"} + } + return Obj{{Key: "data", Value: first}, {Key: "nextCursor", Value: "1"}}, nil + } + p, err := h.lookup(call) + mustDo(t, err) + h.eq([]any{p.Finding, p.Turn.TurnID, strs(p.Older)}) + }) + }) +} + +func itemEntry(turn, kind, text string) Obj { + var item Obj + switch kind { + case "userMessage": + item = Obj{{Key: "type", Value: kind}, {Key: "id", Value: "i"}, {Key: "content", Value: []any{Obj{{Key: "type", Value: "text"}, {Key: "text", Value: text}}}}} + case "commandExecution": + item = Obj{{Key: "type", Value: kind}, {Key: "id", Value: "i"}, {Key: "aggregatedOutput", Value: text}} + case "agentMessage": + item = Obj{{Key: "type", Value: kind}, {Key: "id", Value: "i"}, {Key: "text", Value: text}} + default: + item = Obj{{Key: "type", Value: kind}, {Key: "id", Value: "i"}, {Key: "fragments", Value: []any{Obj{{Key: "text", Value: text}}}}} + } + return Obj{{Key: "turnId", Value: turn}, {Key: "item", Value: item}} +} + +func Test21_HLT29_the_in_turn_and_thread_reads_tell_the_message_from_agent_output(t *testing.T) { + const cls = "TheAdapterLooksBackOnlyToTheSend." + t.Run("the in-turn read asks for that turn's items oldest first", func(t *testing.T) { + mirror(t, hlt, cls+"test_the_in_turn_read_asks_for_that_turns_items_oldest_first", func(h *hl) { + var calls []any + call := func(method string, params Obj) (Obj, error) { + calls = append(calls, []any{method, append(Obj(nil), params...)}) + return Obj{{Key: "data", Value: []any{itemEntry("t1", "userMessage", "requestId: tok")}}, {Key: "nextCursor", Value: nil}}, nil + } + scan, err := BridgeReads{Call: call, Page: 2}.FindTokenInTurn("thread", "tok", "t1", InTurnItemsMax) + mustDo(t, err) + h.eq(scan.Found) + h.eq(calls) + }) + }) + t.Run("the in-turn read counts only that turn's user message", func(t *testing.T) { + mirror(t, hlt, cls+"test_the_in_turn_read_counts_only_that_turns_user_message", func(h *hl) { + call := func(string, Obj) (Obj, error) { + return Obj{{Key: "data", Value: []any{itemEntry("other", "userMessage", "requestId: tok"), itemEntry("t1", "commandExecution", "requestId: tok")}}, {Key: "nextCursor", Value: nil}}, nil + } + scan, err := BridgeReads{Call: call, Page: 8}.FindTokenInTurn("thread", "tok", "t1", InTurnItemsMax) + mustDo(t, err) + h.eq(scan.Found) + h.eq(scan.Exhausted) + }) + }) + t.Run("the thread reads tell the message from agent output", func(t *testing.T) { + mirror(t, hlt, cls+"test_the_thread_reads_tell_the_message_from_agent_output", func(h *hl) { + items := []any{itemEntry("t2", "commandExecution", "del-x-a1"), itemEntry("t2", "hookPrompt", "del-x-a1")} + call := func(string, Obj) (Obj, error) { + return Obj{{Key: "data", Value: items}, {Key: "nextCursor", Value: nil}}, nil + } + b := BridgeReads{Call: call, Page: 8} + only, err := b.FindToken("thread", "del-x-a1", 200, true) + mustDo(t, err) + h.eq(only.Found) + all, err := b.FindToken("thread", "del-x-a1", 200, false) + mustDo(t, err) + h.eq(all.Found) + prompted, err := b.FindTokenSince("thread", "del-x-a1", nil, 200) + mustDo(t, err) + h.eq(prompted.Found) + h.eq([]any{prompted.OtherTurn, prompted.OtherKind}) + items[1] = itemEntry("t2", "fileChange", "del-x-a1") + written, err := b.FindTokenSince("thread", "del-x-a1", nil, 200) + mustDo(t, err) + h.eq(written.Found) + h.eq(written.OtherKind) + items = []any{itemEntry("t2", "someLaterKind", "del-x-a1"), itemEntry("t1", "userMessage", "del-x-a1")} + behind, err := b.FindTokenSince("thread", "del-x-a1", nil, 200) + mustDo(t, err) + h.eq(behind.Found) + }) + }) +} diff --git a/internal/relay/delivery/hostloss_faults_test.go b/internal/relay/delivery/hostloss_faults_test.go new file mode 100644 index 00000000..b58d01b1 --- /dev/null +++ b/internal/relay/delivery/hostloss_faults_test.go @@ -0,0 +1,54 @@ +package delivery + +import ( + "context" + "path/filepath" + "sort" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/faults" +) + +// The fault half of UnknownSendCase: fault_states() sweeps the store and records the batch +// (faultsweep.sweep + record_all), through the todo-21 subset of internal/relay/faults. + +func (h *hl) sweeper() *faults.Sweeper { + return &faults.Sweeper{Store: h.store, MaxAttempts: h.delivery.Policy.MaxAttempts, Now: faults.WallClockISO, + Installation: faults.Installation{Package: "codex-session-relay", Version: "0.1.0", + Location: filepath.Join(repoRoot(h.t), "packages", "codex-session-relay", "src", "codex_session_relay")}, + Current: func(ctx context.Context, event string) (bool, error) { + reason, err := h.delivery.SupersessionReason(ctx, event) + return reason == "", err + }} +} + +// faultStates is fault_states(): the delivery_stalled faults as (state, severity), sorted. +func (h *hl) faultStates() []any { + h.t.Helper() + sw := h.sweeper() + batch, err := sw.Sweep(h.ctx, "crw") + mustDo(h.t, err) + mustDo(h.t, sw.RecordAll(h.ctx, &faults.Ledger{Store: h.store, Clock: h.clock}, batch)) + rows, err := all(h.ctx, h.store, "SELECT state, severity FROM fault_ledger WHERE fault_class = 'delivery_stalled'") + mustDo(h.t, err) + var pairs [][2]string + for _, r := range rows { + pairs = append(pairs, [2]string{r.S("state"), r.S("severity")}) + } + sort.Slice(pairs, func(i, j int) bool { return pairs[i][0]+"|"+pairs[i][1] < pairs[j][0]+"|"+pairs[j][1] }) + out := []any{} + for _, p := range pairs { + out = append(out, []any{p[0], p[1]}) + } + return out +} + +// faultIn is assertIn((state, severity), self.fault_states()). +func (h *hl) faultIn(state, severity string) bool { + for _, p := range h.faultStates() { + pair := p.([]any) + if pair[0] == state && pair[1] == severity { + return true + } + } + return false +} diff --git a/internal/relay/delivery/hostloss_harness_test.go b/internal/relay/delivery/hostloss_harness_test.go new file mode 100644 index 00000000..538eece6 --- /dev/null +++ b/internal/relay/delivery/hostloss_harness_test.go @@ -0,0 +1,805 @@ +package delivery + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "strings" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The Go mirror of tests/test_host_lost_turn.py's HostLossCase (and test_unknown_send_lost.py's +// UnknownSendCase). testdata/capture.py runs every Python test of a module, each in its own tree +// under one root, and records every value the test asserts (the first argument of assertEqual, +// the expression of assertTrue, ...). The Go mirror of a test performs the same steps in the same +// tree, records the value it produced at each of those assertions, and must produce the same +// list; the delivery tables are compared row for row as well. So every asserted value, reason, +// state and next-action word is compared with what Python computed, never with a constant. + +type pyCapture struct { + Captures []any `json:"captures"` + Problems []string `json:"problems"` + Tables map[string][]map[string]any `json:"tables"` + Sends [][]any `json:"sends"` +} + +var ( + captureRoots = map[string]string{} + captureMu sync.Mutex +) + +// pythonCaptures runs capture.py for module once per test process and returns its root. +func pythonCaptures(t *testing.T, module string) string { + t.Helper() + captureMu.Lock() + defer captureMu.Unlock() + if root, ok := captureRoots[module]; ok { + return root + } + root, err := os.MkdirTemp("", "crw-capture-") + mustDo(t, err) + registerCaptureCleanup(root) + repo := repoRoot(t) + script, _ := filepath.Abs("testdata/capture.py") + home, err := os.MkdirTemp("", "crw-capture-home-") + mustDo(t, err) + registerCaptureCleanup(home) + cmd := exec.Command("uv", "run", "--no-sync", "python", script, root, module) + cmd.Dir = filepath.Join(repo, "packages", "codex-session-relay") + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "state"), "XDG_DATA_HOME="+filepath.Join(home, "data"), "XDG_CONFIG_HOME="+filepath.Join(home, "config"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, + "PYTHONPATH="+filepath.Join(repo, "packages", "codex-session-relay", "src")+":"+filepath.Join(repo, "packages", "codex-session-relay")) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("capture %s: %v\n%s", module, err, out) + } + captureRoots[module] = root + return root +} + +var ( + captureCleanups []string + storeAcceptNone = store.AcceptOptions{} +) + +func registerCaptureCleanup(path string) { captureCleanups = append(captureCleanups, path) } + +// hl is one mirrored test: the fixture in the Python test's own tree, the fake host, the +// services, the daemon's in-memory state, and the capture list. +type hl struct { + *fixture + name string + ack *Ack + rc *Reconciler + checks *TurnChecks + adapter Adapter + got []any + policy tickPolicy +} + +// tickPolicy is the part of RetryPolicy the daemon's tick reads. +type tickPolicy struct{ maxTurnChecks, maxSendsTick, maxReconciles int } + +func defaultTick() tickPolicy { return tickPolicy{4, 4, 8} } + +// mirror runs body as the Go twin of module's Class.method and compares it with Python. +func mirror(t *testing.T, module, name string, body func(h *hl)) { + t.Helper() + root := pythonCaptures(t, module) + tree := filepath.Join(root, name) + raw, err := os.ReadFile(filepath.Join(tree, "capture.json")) + mustDo(t, err) + var python pyCapture + mustDo(t, json.Unmarshal(raw, &python)) + if len(python.Problems) > 0 { + t.Fatalf("the Python test itself failed: %s", python.Problems) + } + f := newFixture(t, tree) + f.rid = "" + h := &hl{fixture: f, name: name, ack: NewAck(f.delivery), rc: NewReconciler(f.delivery), adapter: f.host, policy: defaultTick()} + h.checks = &TurnChecks{Reconciler: h.rc, Budget: 4} + body(h) + requireSameCaptures(t, h.got, python.Captures) + requireSameDeliveryTables(t, f, python) + requireSameFaultTables(t, f, python) + requireSameJSON(t, "sends", sendsJSON(f.host), python.Sends) +} + +func requireSameCaptures(t *testing.T, got, want []any) { + t.Helper() + g := normalizeJSON(t, jsonable(got)).([]any) + for i := range g { + g[i] = sameStore(g[i]) + } + w := normalizeJSON(t, want).([]any) + for i := 0; i < max(len(g), len(w)); i++ { + var gi, wi any = "", "" + if i < len(g) { + gi = g[i] + } + if i < len(w) { + wi = w[i] + } + if !reflect.DeepEqual(gi, wi) { + gb, _ := json.Marshal(gi) + wb, _ := json.Marshal(wi) + t.Errorf("assertion %d differs from Python\ngo: %s\npython: %s", i+1, gb, wb) + } + } +} + +// sameStore maps the Go store's directory onto Python's in a captured string: the two stores sit +// side by side in one tree (gostate/ and state/), and a recovery command names its own. +func sameStore(v any) any { + switch x := v.(type) { + case string: + return strings.ReplaceAll(x, string(filepath.Separator)+"gostate", string(filepath.Separator)+"state") + case []any: + for i := range x { + x[i] = sameStore(x[i]) + } + case map[string]any: + for k := range x { + x[k] = sameStore(x[k]) + } + } + return v +} + +// deliveryTables are the tables the delivery, reconciliation and host-loss paths write. The +// daemon's observation pass (poll_observations, its cursors) is todo 29's and is not mirrored. +var deliveryTables = []string{"acks", "ack_evidence", "attempt_messages", "attempts", "deliveries", "events", "failed_operations", "generations", "journal", "reconcile_gate", "recipient_rate", "verdicts"} + +func requireSameDeliveryTables(t *testing.T, f *fixture, python pyCapture) { + t.Helper() + got := normalizeJSON(t, f.tables()).(map[string]any) + want := normalizeJSON(t, python.Tables).(map[string]any) + for _, n := range deliveryTables { + if !reflect.DeepEqual(got[n], want[n]) { + g, _ := json.MarshalIndent(got[n], "", " ") + w, _ := json.MarshalIndent(want[n], "", " ") + t.Errorf("table %s differs from Python\ngo: %s\npython: %s", n, g, w) + } + } +} + +// eq records the value a Python assertEqual/assertNotEqual/assertIsNone/assertTrue asserted. +func (h *hl) eq(values ...any) { + if len(values) == 1 { + h.got = append(h.got, values[0]) + return + } + h.got = append(h.got, values) +} + +// ---------------------------------------------------------------- HostLossCase + +func (h *hl) parentHistory() { + h.host.startTurn(parent, "parent-earlier", "completed", "") + h.clock.Advance(300) +} + +func (h *hl) attemptOn(event string, adapter Adapter, now *float64) Obj { + h.t.Helper() + record, err := h.delivery.Attempt(h.ctx, event, adapter, now, "") + mustDo(h.t, err) + return record +} + +func (h *hl) dispatched() (string, string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + record := h.attemptOn(event, h.host, nil) + h.eq(str(record, "deliveryState")) + return event, str(record, "requestId"), str(record, "turnId") +} + +// completions is HostLossCase.completions: count delivered completions to one parent. +func (h *hl) completions(count int) [][3]string { + h.parentHistory() + var delivered [][3]string + for i := 0; i < count; i++ { + rid := h.register(regOpts{issue: fmt.Sprintf("REL-%d", i+2), dispatchRequest: fmt.Sprintf("dispatch-%d", i+2)}) + h.rid = rid + payload := h.readyPayload(rid, 1, []string{h.artifact(fmt.Sprintf("out-%d.txt", i), fmt.Sprintf("deliverable %d", i))}, 1, assigned("completed")) + _, err := h.accept(payload, storeAcceptNone) + mustDo(h.t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(h.t, err) + record := h.attemptOn(str(payload, "eventId"), h.host, nil) + h.eq(str(record, "deliveryState")) + delivered = append(delivered, [3]string{str(payload, "eventId"), str(record, "requestId"), str(record, "turnId")}) + h.clock.Advance(10) + } + slices.SortFunc(delivered, func(a, b [3]string) int { return strings.Compare(a[0]+a[1]+a[2], b[0]+b[1]+b[2]) }) + return delivered +} + +func (h *hl) acknowledge(event, turn string) { + t := h.host.startTurn(parent, turn, "inProgress", "") + _, err := h.ack.Acknowledge(h.ctx, event, t.TurnID, AckProof(event, t.TurnID), true, nil, h.host) + mustDo(h.t, err) +} + +func (h *hl) reconcile(request string, adapter Adapter) Obj { + h.t.Helper() + out, err := h.rc.ReconcileAttempt(h.ctx, request, adapter, nil) + mustDo(h.t, err) + return out +} + +func (h *hl) tokenConfirmed() (string, string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + turn := h.host.startTurn(parent, "", "completed", "..."+request+"...") + confirmed := h.reconcile(request, h.host) + h.eq(str(confirmed, "evidence")) + h.eq(h.row(event).S("state")) + return event, request, turn.TurnID +} + +func (h *hl) evidenceOf(event string) []any { + var out []any + for _, r := range h.attemptsFor(event) { + out = append(out, r.Opt("affirmative_evidence")) + } + return out +} + +func (h *hl) attemptsFor(event string) []Row { + rows, err := all(h.ctx, h.store, "SELECT * FROM attempts WHERE event_id = ? ORDER BY attempt_no", event) + mustDo(h.t, err) + return rows +} + +func (h *hl) attemptStates(event string) []any { + out := []any{} + for _, r := range h.attemptsFor(event) { + out = append(out, r.Opt("state")) + } + return out +} + +func (h *hl) hostReloadsLosing(turn string) { + h.host.finishTurn(parent, turn, "interrupted") + h.dropItems(turn) +} + +func (h *hl) dropItems(turn string) { + t := h.host.threads[parent] + kept := t.items[:0:0] + for _, item := range t.items { + if item[0] != turn { + kept = append(kept, item) + } + } + t.items = kept +} + +func (h *hl) hostLoses(turn string, items bool) { + t := h.host.threads[parent] + kept := t.turns[:0:0] + for _, one := range t.turns { + if one.TurnID != turn { + kept = append(kept, one) + } + } + t.turns = kept + if items { + h.dropItems(turn) + } +} + +func (h *hl) echo(turn, text string) { + h.host.threads[parent].items = append(h.host.threads[parent].items, [3]string{turn, text, "commandExecution"}) +} + +func (h *hl) item(thread, turn, text, kind string) { + h.host.threads[thread].items = append(h.host.threads[thread].items, [3]string{turn, text, kind}) +} + +func (h *hl) messageOf(turn string) string { + for _, item := range h.host.threads[parent].items { + if item[0] == turn { + return item[1] + } + } + return "" +} + +func (h *hl) uncertainDelivery(message bool) (string, string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.eq(h.row(event).S("state")) + h.clock.Advance(2) + text := "another prompt" + if message { + text = "[codex-session-relay] verification request\nrequestId: " + request + } + turn := h.host.startTurn(parent, "", "inProgress", text) + return event, request, turn.TurnID +} + +func (h *hl) foldedDelivery(message bool, earlier, later int) (string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.startTurn(parent, "folded", "inProgress", "") + for n := 0; n < earlier; n++ { + h.item(parent, "folded", fmt.Sprintf("earlier work %d", n), "commandExecution") + } + h.clock.Advance(120) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.eq(h.row(event).S("state")) + if message { + h.item(parent, "folded", "[codex-session-relay] verification request\nrequestId: "+request, "") + } + for n := 0; n < later; n++ { + h.item(parent, "folded", fmt.Sprintf("later work %d", n), "commandExecution") + } + return event, request +} + +func (h *hl) cliAck(event, turn string, adapter Adapter) Obj { + h.t.Helper() + if adapter == nil { + adapter = h.host + } + out, err := AckCommand(h.ctx, h.ack, h.rc, adapter, event, turn, AckProof(event, turn), nil) + mustDo(h.t, err) + return out +} + +func (h *hl) cliVerdict(event, turn string) Obj { + h.t.Helper() + mustDo(h.t, CompleteKeptAcknowledgement(h.ctx, h.ack, h.rc, h.host, event)) + out, err := h.ack.RecordVerdict(h.ctx, event, "verified", turn, nil, nil, nil, nil) + mustDo(h.t, err) + return out +} + +func (h *hl) ackRow(event string) Row { + return h.one("SELECT a.verified, e.last_reason FROM acks a LEFT JOIN ack_evidence e ON e.event_id = a.event_id WHERE a.event_id = ?", event) +} + +func (h *hl) statusOf(event string) Obj { + h.t.Helper() + item, err := h.delivery.SnapshotItem(h.ctx, event) + mustDo(h.t, err) + return item +} + +func (h *hl) statusPair(event string) []any { + item := h.statusOf(event) + p, _ := get(item, "phase") + r, _ := get(item, "reported") + return []any{p, r} +} + +func (h *hl) journalled(kind string) int64 { + return h.count("SELECT COUNT(*) AS c FROM journal WHERE kind = ?", kind) +} + +func (h *hl) exec(query string, args ...any) { + h.t.Helper() + mustDo(h.t, h.store.Transaction(h.ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := execSQL(ctx, h.store, query, args...) + return err + })) +} + +func field(o Obj, key string) any { + v, _ := get(o, key) + return v +} + +func sub(o Obj, key string) Obj { + v, _ := get(o, key) + s, _ := v.(Obj) + return s +} + +// ---------------------------------------------------------------- the tick + +// tickReport is daemon.TickReport for the passes this package owns; the others stay 0. +type tickReport struct { + reconciled, delivered, deferred, skipped, acksVerified, anchorsBound, turnsLost, turnsUndecided int + notes []string +} + +func (r tickReport) quiet() bool { + return r.reconciled+r.delivered+r.deferred+r.acksVerified+r.anchorsBound+r.turnsLost+r.turnsUndecided == 0 +} + +func (r tickReport) asDict() Obj { + return Obj{{Key: "observed", Value: 0}, {Key: "reconciled", Value: r.reconciled}, {Key: "delivered", Value: r.delivered}, {Key: "deferred", Value: r.deferred}, + {Key: "skipped", Value: r.skipped}, {Key: "acksVerified", Value: r.acksVerified}, {Key: "anchorsBound", Value: r.anchorsBound}, {Key: "requeued", Value: 0}, + {Key: "faultsRecorded", Value: 0}, {Key: "supervisorStaged", Value: 0}, {Key: "supervisorSent", Value: 0}, {Key: "notificationsDelivered", Value: 0}, + {Key: "turnsLost", Value: r.turnsLost}, {Key: "turnsUndecided", Value: r.turnsUndecided}, {Key: "quiet", Value: r.quiet()}} +} + +// tick is RelayDaemon.tick over the passes this package ports, in its order: bind anchors, +// reconcile, bind again, verify acknowledgements (kept ones confirmed first), check delivered +// turns, deliver. Observation of child turns, requeue, the fault sweep and the supervisor pass +// (todo 29, 22, 24) have nothing to do in these scenarios and are not run. +func (h *hl) tickWith(policy tickPolicy, adapter Adapter, checks *TurnChecks) tickReport { + h.t.Helper() + now := h.clock.Now() + var r tickReport + bind := func() { + bound, err := h.ack.BindPendingAnchors(h.ctx) + if err != nil { + r.notes = append(r.notes, "anchor recovery failed: "+err.Error()) + return + } + r.anchorsBound += len(bound) + } + bind() + var rr ReconcileReport + mustDo(h.t, ReconcilePass(h.ctx, h.rc, adapter, policy.maxReconciles, now, &rr)) + r.reconciled, r.skipped = rr.Reconciled, rr.Skipped + bind() + r.notes = append(r.notes, ConfirmKeptAcks(h.ctx, h.ack, h.rc, adapter, now)...) + results, err := h.ack.VerifyPendingAcks(h.ctx, adapter, 8, &now) + mustDo(h.t, err) + for _, one := range results { + if str(one.(Obj), "outcome") == "verified" { + r.acksVerified++ + } + } + checks.Budget = policy.maxTurnChecks + var tc TurnCheckReport + checks.Pass(h.ctx, adapter, now, &tc) + r.turnsLost, r.turnsUndecided = tc.TurnsLost, tc.TurnsUndecided + sends := policy.maxSendsTick + if sends == 0 { + sends = -1 + } + var counts TickCounts + mustDo(h.t, (&Scheduler{Delivery: h.delivery, Ack: h.ack, MaxSendsTick: sends}).Deliver(h.ctx, adapter, now, &counts)) + r.delivered, r.deferred, r.skipped = counts.Delivered, counts.Deferred, r.skipped+counts.Skipped + return r +} + +func (h *hl) tick() tickReport { return h.tickWith(h.policy, h.adapter, h.checks) } + +// ---------------------------------------------------------------- assignment-show + +// nextAction is AssignmentView.state(rid)["nextExpectedAction"] for the states these tests +// reach, read over the same projection statement (Anchored): correction_next_action, then +// completion_next_action, then NEXT_ACTION. The assignment view itself is todo 25's +// (registry.CompletionNextAction on crw-154); this reads what delivery owns. +func (h *hl) nextAction() string { return str(h.assignment(), "nextExpectedAction") } + +func (h *hl) assignment() Obj { + h.t.Helper() + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(h.t, err) + head, err := HeadRevision(h.ctx, h.store, h.rid, r.Generation) + mustDo(h.t, err) + headID, _ := get(head, "eventId") + var verdict Row + if headID != nil { + verdict = h.one("SELECT verdict FROM verdicts WHERE event_id = ?", headID) + } + previous := h.one("SELECT v.event_id FROM verdicts v JOIN events e ON e.event_id = v.event_id WHERE e.relationship_id = ? AND e.execution_generation < ? AND v.verdict = 'needs_changes' ORDER BY e.execution_generation DESC LIMIT 1", h.rid, r.Generation) + state := "requested" + switch { + case verdict != nil && verdict.S("verdict") == "verified": + state = "verified" + case headID != nil && previous != nil: + state = "corrected" + case headID != nil && h.one("SELECT 1 FROM verification_claims WHERE event_id = ?", headID) != nil: + state = "verifying" + case headID != nil: + state = "received" + case previous != nil: + state = "needs_changes" + } + projection := h.projection(r.Generation, headID) + action := correctionNextAction(state, projection) + if action == "" { + action = completionNextAction(state, projection) + } + if action == "" { + action = map[string]string{"requested": "child_emits", "received": "daemon_delivers", "verifying": "parent_verifies", "needs_changes": "child_corrects", "corrected": "parent_verifies", "verified": "coordinator_integrates"}[state] + } + out := Obj{{Key: "state", Value: state}, {Key: "nextExpectedAction", Value: action}, {Key: "projection", Value: projection}} + if recovery := h.parentRecovery(action, projection); recovery != nil { + out = append(out, F{Key: "recovery", Value: recovery}) + } + return out +} + +func (h *hl) projection(generation int64, headID any) Obj { + anchored := func(event any) Obj { + out, err := Anchored(h.ctx, h.delivery, event, generation) + if err != nil && strings.Contains(err.Error(), "settings-hold reading") { + // withheld_pre_send / inbox_only: the settings-hold reading is todo 25's; these + // scenarios stage no settings refusal, so Python reads no settings hold either. + return h.anchoredWithoutSettings(event, generation) + } + mustDo(h.t, err) + return out + } + correction := h.one("SELECT event_id FROM events WHERE relationship_id = ? AND execution_generation = ? AND outcome = 'revision_request' AND suppressed_reason IS NULL ORDER BY event_id LIMIT 1", h.rid, generation) + var correctionID any + if correction != nil { + correctionID = correction.S("event_id") + } + return Obj{{Key: "completion", Value: anchored(headID)}, {Key: "correction", Value: anchored(correctionID)}} +} + +func (h *hl) anchoredWithoutSettings(event any, generation int64) Obj { + row := h.one("SELECT d.state, d.hold_reason, (SELECT COUNT(*) FROM attempts x WHERE x.event_id = d.event_id AND x.state = 'host_lost_turn') AS lost, k.event_id AS acked, k.verified, k.accepted, v.last_reason FROM deliveries d LEFT JOIN acks k ON k.event_id = d.event_id LEFT JOIN ack_evidence v ON v.event_id = d.event_id WHERE d.event_id = ?", event) + delivery := Obj{{Key: "state", Value: row.S("state")}, {Key: "holdReason", Value: row.Opt("hold_reason")}, {Key: "hostLostAttempts", Value: row.I("lost")}, {Key: "pacing", Value: nil}, {Key: "settingsHold", Value: nil}} + ack := Obj{{Key: "settlement", Value: nil}} + if !row.N("acked") { + ack = Obj{{Key: "accepted", Value: row.I("accepted") != 0}, {Key: "settlement", Value: row.Opt("verified")}, {Key: "lastReason", Value: row.Opt("last_reason")}} + } + var reason any + if row.S("hold_reason") != "" { + reason = Obj{{Key: "source", Value: "deliveries.hold_reason"}, {Key: "value", Value: row.S("hold_reason")}} + } + return Obj{{Key: "eventId", Value: event}, {Key: "executionGeneration", Value: generation}, {Key: "delivery", Value: delivery}, {Key: "ack", Value: ack}, {Key: "undeliveredReason", Value: reason}, {Key: "supersession", Value: nil}} +} + +func neverReopens(delivery Obj) bool { + p := sub(delivery, "pacing") + return p != nil && str(p, "reason") == HourlyCap && field(p, "reopensAt") == nil +} + +func correctionNextAction(state string, projection Obj) string { + correction := sub(projection, "correction") + delivery := sub(correction, "delivery") + if state != "needs_changes" || delivery == nil { + return "" + } + dstate := str(delivery, "state") + if field(correction, "supersession") != nil || dstate == Superseded { + return correctionAnswered + } + if dstate == Dispatched || dstate == Acknowledged { + return "" + } + if dstate == InboxOnly || str(sub(correction, "undeliveredReason"), "source") == "deliveries.hold_reason" { + return correctionHeld + } + if dstate == Sending || dstate == HeldUncertain { + return correctionUnconfirmed + } + if slices.Contains(claimable, dstate) { + if neverReopens(delivery) { + return "operator_changes_send_policy" + } + return "daemon_delivers_correction" + } + return "" +} + +func completionNextAction(state string, projection Obj) string { + if state != "received" && state != "corrected" && state != "verifying" { + return "" + } + completion := sub(projection, "completion") + delivery := sub(completion, "delivery") + if delivery == nil { + return "" + } + ack := sub(completion, "ack") + lost := field(delivery, "hostLostAttempts").(int64) > 0 + settlement := field(ack, "settlement") + if settlement == "verified" { + if state == "received" && field(ack, "accepted") == true { + return "parent_verifies" + } + return "" + } + if hold := str(delivery, "holdReason"); hold != "" && hold != PushChannelClosed { + switch hold { + case HostLostTurn: + return "parent_recovers_host_lost_turn" + case UnknownSendLost: + return unknownSendHeldAction + case UnknownSendUndecided: + return unknownSendUndecidedAction + } + if lost { + return "parent_recovers_host_lost_turn" + } + return "" + } + switch str(delivery, "state") { + case HeldUncertain, Sending: + return reconcileAction + case Dispatched, InboxOnly: + if settlement != nil { + switch field(ack, "lastReason") { + case nil, "unverified_turn", DeliveryUnconfirmed: + return "daemon_verifies_acknowledgement" + } + return "parent_reacknowledges" + } + return "parent_acknowledges" + case Queued, DeferredBusy, WithheldPreSend: + if neverReopens(delivery) { + return "operator_changes_send_policy" + } + if lost { + return "daemon_redelivers_host_lost_turn" + } + return "daemon_delivers" + } + return "" +} + +func (h *hl) parentRecovery(action string, projection Obj) Obj { + var anchored Obj + switch action { + case correctionHeld: + anchored = sub(projection, "correction") + case "parent_recovers_host_lost_turn", unknownSendHeldAction, unknownSendUndecidedAction: + anchored = sub(projection, "completion") + default: + return nil + } + delivery := sub(anchored, "delivery") + reason := field(delivery, "holdReason") + if !truthy(reason) { + reason = field(delivery, "state") + } + directory, _ := filepath.Abs(filepath.Dir(h.store.Path)) + return Obj{{Key: "actor", Value: "parent"}, {Key: "reason", Value: reason}, {Key: "command", Value: recoveryCommand(directory, pyStr(field(anchored, "eventId")))}, {Key: "then", Value: parentRecoveryThen}} +} + +func (h *hl) completionDelivery() Obj { + return sub(sub(sub(h.assignment(), "projection"), "completion"), "delivery") +} + +// ---------------------------------------------------------------- wrapped adapters + +// hooked is a fake host with some reads or the send replaced, everything else passed through +// (the Python tests' delegating wrappers). +type hooked struct { + Adapter + findDispatched func(thread, turn string, sentAt float64) (TurnPresence, error) + findToken func(thread, token string, limit int, messageOnly bool) (TokenScan, error) + readTurn func(thread, turn string) (*TurnInfo, error) + send func(requestID, thread, message string, settings *TaskSettings) (Obj, error) + getOperation func(requestID string) (Obj, error) + findInTurn func(thread, token, turnID string, limit int) (TokenScan, error) +} + +func (w *hooked) FindDispatchedTurn(thread, turn string, sentAt float64) (TurnPresence, error) { + if w.findDispatched != nil { + return w.findDispatched(thread, turn, sentAt) + } + return w.Adapter.FindDispatchedTurn(thread, turn, sentAt) +} + +func (w *hooked) FindToken(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + if w.findToken != nil { + return w.findToken(thread, token, limit, messageOnly) + } + return w.Adapter.FindToken(thread, token, limit, messageOnly) +} + +func (w *hooked) ReadTurn(thread, turn string) (*TurnInfo, error) { + if w.readTurn != nil { + return w.readTurn(thread, turn) + } + return w.Adapter.ReadTurn(thread, turn) +} + +func (w *hooked) SendMessage(requestID, thread, message string, settings *TaskSettings) (Obj, error) { + if w.send != nil { + return w.send(requestID, thread, message, settings) + } + return w.Adapter.SendMessage(requestID, thread, message, settings) +} + +func (w *hooked) GetOperation(requestID string) (Obj, error) { + if w.getOperation != nil { + return w.getOperation(requestID) + } + return w.Adapter.GetOperation(requestID) +} + +func (w *hooked) FindTokenInTurn(thread, token, turnID string, limit int) (TokenScan, error) { + if w.findInTurn != nil { + return w.findInTurn(thread, token, turnID, limit) + } + return w.Adapter.FindTokenInTurn(thread, token, turnID, limit) +} + +// countingLookups is CountingLookups: every recipient-turn lookup, by turn id. +func countingLookups(inner Adapter, lookups *[]string) *hooked { + return &hooked{Adapter: inner, findDispatched: func(thread, turn string, sentAt float64) (TurnPresence, error) { + *lookups = append(*lookups, turn) + return inner.FindDispatchedTurn(thread, turn, sentAt) + }} +} + +// countingReads is CountingReads: the receipt and item reads a confirmation could make. +func countingReads(inner Adapter, reads *[]any) *hooked { + return &hooked{Adapter: inner, + getOperation: func(id string) (Obj, error) { + *reads = append(*reads, []any{"get_operation", id}) + return inner.GetOperation(id) + }, + findToken: func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + *reads = append(*reads, []any{"find_token", token}) + return inner.FindToken(thread, token, limit, messageOnly) + }, + findInTurn: func(thread, token, turnID string, limit int) (TokenScan, error) { + *reads = append(*reads, []any{"find_token_in_turn", turnID}) + return inner.FindTokenInTurn(thread, token, turnID, limit) + }} +} + +// actsDuringTheTurnRead is ActsDuringTheTurnRead: something else runs first, once. +func actsDuringTheTurnRead(inner Adapter, action func()) *hooked { + return &hooked{Adapter: inner, readTurn: func(thread, turn string) (*TurnInfo, error) { + if action != nil { + a := action + action = nil + a() + } + return inner.ReadTurn(thread, turn) + }} +} + +func preSendRejection(request string) Obj { + return Obj{{Key: "requestId", Value: request}, {Key: "status", Value: "failed"}, {Key: "error", Value: "thread/read: transport refused"}, {Key: "rpcError", Value: Obj{{Key: "code", Value: "internal"}, {Key: "message", Value: "refused"}}}} +} + +// requireSameFaultTables compares the delivery_stalled rows of the fault tables; the other +// classes come from sources todo 22 and todo 29 own (observation_stalled needs the daemon's +// poll rows), and fault_cursors carries the wall clock. +func requireSameFaultTables(t *testing.T, f *fixture, python pyCapture) { + t.Helper() + got := normalizeJSON(t, f.tables()).(map[string]any) + want := normalizeJSON(t, python.Tables).(map[string]any) + stalled := func(tables map[string]any) map[string]bool { + ids := map[string]bool{} + rows, _ := tables["fault_ledger"].([]any) + for _, r := range rows { + if m := r.(map[string]any); m["fault_class"] == "delivery_stalled" { + ids[m["fault_id"].(string)] = true + } + } + return ids + } + keep := func(tables map[string]any, name string, ids map[string]bool) []any { + out := []any{} + rows, _ := tables[name].([]any) + for _, r := range rows { + m := r.(map[string]any) + if ids[fmt.Sprint(m["fault_id"])] { + delete(m, "seq") + out = append(out, m) + } + } + return out + } + gids, wids := stalled(got), stalled(want) + for _, name := range []string{"fault_ledger", "fault_occurrences", "fault_timeline", "fault_publications", "fault_notifications"} { + g, w := keep(got, name, gids), keep(want, name, wids) + if !reflect.DeepEqual(g, w) { + gb, _ := json.MarshalIndent(g, "", " ") + wb, _ := json.MarshalIndent(w, "", " ") + t.Errorf("table %s (delivery_stalled) differs from Python\ngo: %s\npython: %s", name, gb, wb) + } + } +} diff --git a/internal/relay/delivery/intent.go b/internal/relay/delivery/intent.go new file mode 100644 index 00000000..32d3f8ae --- /dev/null +++ b/internal/relay/delivery/intent.go @@ -0,0 +1,945 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "math" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Management intent (intent.py): what a coordinator publishes before the task it is about +// exists. The assignment state is DERIVED from which facts exist, never stored. + +// BindingWindowMinutes is BINDING_WINDOW_MINUTES. +const BindingWindowMinutes = 30 + +// Derived assignment states and create-once outcomes (intent.py). +const ( + RelationshipRegistered = "relationship_registered" + IdentityBound = "identity_bound" + AmbiguousIdentity = "ambiguous_identity" + IntentExpired = "intent_expired" + CreationUnknown = "creation_unknown" + CreationAccepted = "creation_accepted" + IntentDeclared = "intent_declared" + + Bound = "bound" + Unchanged = "unchanged" + Conflict = "conflict" + + DispatchCurrent = "current" + DispatchStale = "stale" + DispatchAbsent = "absent" +) + +// Refusal reasons the marker writers raise that the delivery package did not already name. +const ( + RelationshipConflict = "relationship_conflict" + UnboundGeneration = "unbound_generation" + OutcomeInconsistent = "outcome_inconsistent" +) + +// AttemptOutcomes is ATTEMPT_OUTCOMES. +var AttemptOutcomes = []string{"accepted", "unknown", "failed"} + +// DispositionOutcomes is DISPOSITION_OUTCOMES: exhaustive, not illustrative. +var DispositionOutcomes = []string{"in_progress", "blocked_needs_input", "interrupted", "failed", "ready_for_review"} + +// intentFields is INTENT_FIELDS: everything an intent means, which is everything but when. +var intentFields = []string{"dispatchRequestIdHash", "issueKey", "workspace", "dbPath", "criteriaSource", "baselineRevision", "authorizedSettings"} + +var singleKeys = []string{"intent", "bound", "relationship"} +var listedKeys = []string{"attempts", "claims", "conflicts", "resolutions"} + +// identityFields is IDENTITY_FIELDS: every identity slot a fact may carry, in Python's order. +var identityFields = map[string][]string{ + "intent": {"dispatchRequestIdHash", "dbPath"}, + "bound": {"sessionId", "taskId"}, + "relationship": {"relationshipId"}, + "attempts": {"taskId", "outcome"}, + "claims": {"sessionId", "dispatchRequestId"}, + "conflicts": {"attemptedSessionId", "attemptedTaskId"}, + "resolutions": {"chosenTaskId", "chosenSessionId"}, +} + +// numberFields is NUMBER_FIELDS: a present value must be a positive integer. +var numberFields = map[string][]string{"relationship": {"executionGeneration"}} + +var isoOffset = regexp.MustCompile(`([+-]\d{2}):?(\d{2})(?::?(\d{2}))?$`) + +// Moment is intent.moment: datetime.fromisoformat, "Z" read as UTC and a naive stamp as UTC; +// nil for a value that is not a timestamp. +func Moment(value any) *time.Time { + text, ok := value.(string) + if !ok { + if value == nil { + return nil + } + text = pyStr(value) + } + text = strings.Replace(text, "Z", "+00:00", 1) + zone := time.UTC + if m := isoOffset.FindStringSubmatchIndex(text); m != nil && len(text) > 10 { + sign, hh, mm := text[m[2]], text[m[2]+1:m[3]], text[m[4]:m[5]] + ss := "00" + if m[6] >= 0 { + ss = text[m[6]:m[7]] + } + h, _ := strconv.Atoi(hh) + mi, _ := strconv.Atoi(mm) + se, _ := strconv.Atoi(ss) + offset := h*3600 + mi*60 + se + if sign == '-' { + offset = -offset + } + zone = time.FixedZone("", offset) + text = text[:m[0]] + } + for _, layout := range []string{"2006-01-02T15:04:05.999999999", "2006-01-02 15:04:05.999999999", "2006-01-02T15:04", "2006-01-02 15:04", "2006-01-02T15", "2006-01-02"} { + if t, err := time.ParseInLocation(layout, text, zone); err == nil { + return &t + } + } + return nil +} + +// ---------------------------------------------------------------- shape + +// Malformed is malformed: the first published record that is not the shape a fact must be. +func Malformed(marker Obj) string { + for _, key := range singleKeys { + if v, ok := get(marker, key); ok { + if _, record := v.(Obj); !record { + return key + } + } + } + for _, key := range listedKeys { + v, ok := get(marker, key) + if !ok { + continue + } + items, list := v.([]any) + if !list { + return key + } + for _, item := range items { + if _, record := item.(Obj); !record { + return key + } + } + } + for _, key := range append(append([]string{}, singleKeys...), listedKeys...) { + v, ok := get(marker, key) + if !ok { + continue + } + items := []any{v} + if list, isList := v.([]any); isList { + items = list + } + for _, raw := range items { + item := raw.(Obj) + for _, field := range identityFields[key] { + if value, present := get(item, field); present { + if _, text := value.(string); !text { + return key + "." + field + } + } + } + for _, field := range numberFields[key] { + value, present := get(item, field) + if !present { + continue + } + if n, integer := value.(int64); !integer || n < 1 { + return key + "." + field + } + } + if key == "resolutions" { + if entries, present := get(item, "adjudicated"); present { + list, isList := entries.([]any) + if !isList { + return key + ".adjudicated" + } + for _, entry := range list { + if _, record := entry.(Obj); !record { + return key + ".adjudicated" + } + } + } + } + } + } + return "" +} + +// MalformedCounters is malformed_counters: a count that is not a count is corruption. +func MalformedCounters(counters any) string { + if counters == nil { + return "" + } + record, ok := counters.(Obj) + if !ok { + return "counters" + } + for _, f := range record { + if n, integer := f.Value.(int64); !integer || n < 0 { + return "counters." + f.Key + } + } + return "" +} + +// ---------------------------------------------------------------- coverage + +func objects(items []any) []Obj { + out := make([]Obj, 0, len(items)) + for _, item := range items { + record, _ := item.(Obj) + out = append(out, record) + } + return out +} + +func resolutionsOf(marker Obj) []Obj { return objects(markerFactList(marker, "resolutions")) } + +// FactCovered is covered: this exact fact, by identity AND digest, adjudicated by a resolution. +func FactCovered(fact Obj, resolutions []Obj) bool { + factID := fieldOf(fact, "factId") + if !Named(factID) { + return false + } + digest := FactDigest(fact) + for _, resolution := range resolutions { + entries, _ := fieldOf(resolution, "adjudicated").([]any) + for _, raw := range entries { + entry, ok := raw.(Obj) + if !ok { + continue + } + if fieldOf(entry, "factId") == factID && fieldOf(entry, "digest") == digest { + return true + } + } + } + return false +} + +// Claimant is claimant: the session a claim belongs to, from the path that authorised the write, +// and only when the body names that same session. nil when it owns nothing. +func Claimant(claim Obj) any { + parts := strings.Split(pyStrOr(fieldOf(claim, "factId")), "/") + if len(parts) != 3 || parts[0] != "claims" || parts[2] != claimFile { + return nil + } + owner := parts[1] + if owner == "." || owner == ".." || !Named(owner) { + return nil + } + body := fieldOf(claim, "sessionId") + if !Named(body) || body != owner { + return nil + } + return owner +} + +// pyStrOr is str(value or ""): Python's falsy values read as the empty string. +func pyStrOr(value any) string { + if !truthy(value) { + return "" + } + return pyStr(value) +} + +func acceptedTasks(marker Obj) map[any]bool { + tasks := map[any]bool{} + for _, attempt := range objects(markerFactList(marker, "attempts")) { + if fieldOf(attempt, "outcome") == "accepted" && Named(fieldOf(attempt, "taskId")) { + tasks[fieldOf(attempt, "taskId")] = true + } + } + return tasks +} + +func competingFacts(marker Obj) []Obj { + bound := markerFact(marker, "bound") + var facts []Obj + for _, attempt := range objects(markerFactList(marker, "attempts")) { + if fieldOf(attempt, "outcome") == "accepted" && Named(fieldOf(attempt, "taskId")) && !SameIdentity(fieldOf(attempt, "taskId"), fieldOf(bound, "taskId")) { + facts = append(facts, attempt) + } + } + for _, claim := range objects(markerFactList(marker, "claims")) { + if !SameIdentity(Claimant(claim), fieldOf(bound, "sessionId")) { + facts = append(facts, claim) + } + } + return append(facts, objects(markerFactList(marker, "conflicts"))...) +} + +func ambiguityResolved(marker Obj) bool { + var resolutions []Obj + for _, r := range resolutionsOf(marker) { + if Named(fieldOf(r, "chosenTaskId")) && Named(fieldOf(r, "chosenSessionId")) { + resolutions = append(resolutions, r) + } + } + if len(resolutions) == 0 { + return false + } + pairs := map[[2]any]bool{} + for _, r := range resolutions { + pairs[[2]any{fieldOf(r, "chosenTaskId"), fieldOf(r, "chosenSessionId")}] = true + } + if len(pairs) != 1 { + return false + } + var chosen [2]any + for pair := range pairs { + chosen = pair + } + sessions := map[any]bool{} + for _, claim := range objects(markerFactList(marker, "claims")) { + if owner := Claimant(claim); owner != nil { + sessions[owner] = true + } + } + if !acceptedTasks(marker)[chosen[0]] || !sessions[chosen[1]] { + return false + } + var facts []Obj + for _, attempt := range objects(markerFactList(marker, "attempts")) { + if fieldOf(attempt, "outcome") == "accepted" && Named(fieldOf(attempt, "taskId")) { + facts = append(facts, attempt) + } + } + facts = append(facts, objects(markerFactList(marker, "claims"))...) + for _, fact := range facts { + if !FactCovered(fact, resolutions) { + return false + } + } + return true +} + +// IdentityContested is identity_contested: a competing fact after a bind that no resolution +// naming the BOUND identity has adjudicated. +func IdentityContested(marker Obj) bool { + bound := markerFact(marker, "bound") + if len(bound) == 0 { + return false + } + var applicable []Obj + for _, r := range resolutionsOf(marker) { + if SameIdentity(fieldOf(r, "chosenSessionId"), fieldOf(bound, "sessionId")) && SameIdentity(fieldOf(r, "chosenTaskId"), fieldOf(bound, "taskId")) { + applicable = append(applicable, r) + } + } + for _, fact := range competingFacts(marker) { + if !FactCovered(fact, applicable) { + return true + } + } + return false +} + +// ---------------------------------------------------------------- state + +// DeriveAssignmentState is derive_assignment_state: the contract's precedence table, in order. +func DeriveAssignmentState(marker Obj, now any) string { + intent := markerFact(marker, "intent") + attempts := objects(markerFactList(marker, "attempts")) + claims := markerFactList(marker, "claims") + if len(markerFact(marker, "bound")) > 0 { + if Named(fieldOf(markerFact(marker, "relationship"), "relationshipId")) { + return RelationshipRegistered + } + return IdentityBound + } + resolvedAmbiguity := ambiguityResolved(marker) + accepted, unknown := false, false + for _, a := range attempts { + switch fieldOf(a, "outcome") { + case "accepted": + accepted = true + case "unknown": + unknown = true + } + } + if !resolvedAmbiguity && (len(acceptedTasks(marker)) > 1 || len(claims) > 1) { + return AmbiguousIdentity + } + anchor, seen := Moment(fieldOf(intent, "declaredAt")), Moment(now) + if anchor != nil && seen != nil && seen.After(anchor.Add(BindingWindowMinutes*time.Minute)) { + return IntentExpired + } + if !resolvedAmbiguity && !accepted && unknown { + return CreationUnknown + } + if accepted { + return CreationAccepted + } + return IntentDeclared +} + +// Why a session's claim does not correlate (intent.py labels). +const ( + ClaimAbsent = "claim_absent" + ClaimDispatchUnnamed = "claim_dispatch_unnamed" + ClaimDispatchMismatch = "claim_dispatch_mismatch" + IntentDispatchUnnamed = "intent_dispatch_unnamed" + IntentAssignmentMismatch = "intent_assignment_mismatch" +) + +func ownClaim(marker Obj, sessionID any) Obj { + for _, claim := range objects(markerFactList(marker, "claims")) { + if SameIdentity(Claimant(claim), sessionID) { + return claim + } + } + return nil +} + +// CorrelationProblem is correlation_problem: which condition this session's claim fails, or "". +func CorrelationProblem(marker Obj, sessionID, assignment any) string { + claim := ownClaim(marker, sessionID) + if len(claim) == 0 { + return ClaimAbsent + } + presented := fieldOf(claim, "dispatchRequestId") + if !Named(presented) { + return ClaimDispatchUnnamed + } + declared := fieldOf(markerFact(marker, "intent"), "dispatchRequestIdHash") + if !Named(declared) { + return IntentDispatchUnnamed + } + if Named(assignment) && !SameIdentity(declared, assignment) { + return IntentAssignmentMismatch + } + if SameIdentity(AssignmentID(presented.(string)), declared) { + return "" + } + return ClaimDispatchMismatch +} + +// Correlated is correlated: correlation_problem read as a yes or no. +func Correlated(marker Obj, sessionID, assignment any) bool { + return CorrelationProblem(marker, sessionID, assignment) == "" +} + +// ClaimedDispatch is claimed_dispatch: the dispatch this session claimed, where that correlates. +func ClaimedDispatch(marker Obj, sessionID, assignment any) any { + if CorrelationProblem(marker, sessionID, assignment) != "" { + return nil + } + presented := fieldOf(ownClaim(marker, sessionID), "dispatchRequestId") + if Named(presented) { + return presented + } + return nil +} + +// ---------------------------------------------------------------- selection + +func obstructedClaim(marker Obj, sessionID any) Obj { + for _, raw := range markerFactList(marker, "claims") { + claim, ok := raw.(Obj) + if !ok || !SameIdentity(Claimant(claim), sessionID) { + continue + } + if value, present := get(claim, "dispatchRequestId"); present { + if _, text := value.(string); !text { + return claim + } + } + } + return nil +} + +func selectingClaim(marker Obj, sessionID any, assignment string) Obj { + for _, raw := range markerFactList(marker, "claims") { + claim, ok := raw.(Obj) + if !ok || !SameIdentity(Claimant(claim), sessionID) { + continue + } + presented := fieldOf(claim, "dispatchRequestId") + if !Named(presented) || !SameIdentity(AssignmentID(presented.(string)), assignment) { + continue + } + intentValue, _ := get(marker, "intent") + intent, readable := intentValue.(Obj) + if readable { + for _, field := range identityFields["intent"] { + if value, present := get(intent, field); present { + if _, text := value.(string); !text { + readable = false + } + } + } + } + var declared any + if readable { + declared = fieldOf(intent, "dispatchRequestIdHash") + } + if Named(declared) && !SameIdentity(declared, assignment) { + continue + } + return claim + } + return nil +} + +type candidate struct { + directory string + facts Obj + problems []string +} + +// recencyLess is _recency's ordering: a readable declaration above none, then the instant, then +// the assignment id. +func recencyLess(a, b candidate) bool { + moment := func(c candidate) *time.Time { + if intent, ok := fieldOfAny(c.facts, "intent").(Obj); ok { + return Moment(fieldOf(intent, "declaredAt")) + } + return nil + } + ma, mb := moment(a), moment(b) + if (ma != nil) != (mb != nil) { + return ma == nil + } + if ma != nil && !ma.Equal(*mb) { + return ma.Before(*mb) + } + return filepath.Base(a.directory) < filepath.Base(b.directory) +} + +func fieldOfAny(o Obj, key string) any { v, _ := get(o, key); return v } + +// SelectAssignment is select_assignment: which assignment under this workspace a session's turn +// is about. A claim naming THIS assignment is consulted before recency. It returns ("", nil, +// ["workspace"]) when the workspace could not be listed and ("", nil, []) when nothing is +// selectable. +func SelectAssignment(root, workspace string, sessionID any) (string, Obj, []string, error) { + listed, readable, err := ListAssignments(root, workspace) + if err != nil { + return "", nil, nil, err + } + if !readable { + return "", nil, []string{"workspace"}, nil + } + var candidates []candidate + for _, directory := range listed { + facts, problems := ReadAssignment(directory) + _, hasIntent := get(facts, "intent") + unreadableIntent := false + for _, p := range problems { + if p == "intent" { + unreadableIntent = true + } + } + if !hasIntent && !unreadableIntent { + continue + } + candidates = append(candidates, candidate{directory, facts, problems}) + } + if len(candidates) == 0 { + return "", nil, []string{}, nil + } + var claimed []candidate + for _, c := range candidates { + if selectingClaim(c.facts, sessionID, filepath.Base(c.directory)) != nil || obstructedClaim(c.facts, sessionID) != nil { + claimed = append(claimed, c) + } + } + pool := candidates + if len(claimed) > 0 { + pool = claimed + } + best := pool[0] + for _, c := range pool[1:] { + if recencyLess(best, c) { + best = c + } + } + return best.directory, best.facts, best.problems, nil +} + +// ---------------------------------------------------------------- writing + +func registrationError(reason, detail string) error { + return &store.RefusedError{Reason: reason, Detail: detail} +} + +func checkedAssignment(value any) (string, error) { + if !ValidAssignment(value) { + return "", registrationError(UnknownGeneration, "an assignment id is the hex sha256 of a dispatch request id, not "+pyReprValue(value)) + } + return value.(string), nil +} + +func checkedIdentity(value any, what string) (string, error) { + if !ValidSegment(value) { + return "", registrationError(UnboundGeneration, "a "+what+" becomes a directory name, so it cannot be empty, . or .., or contain a path separator: "+pyReprValue(value)) + } + return value.(string), nil +} + +func assignmentDirectory(root, workspace string, assignment any) (string, error) { + checked, err := checkedAssignment(assignment) + if err != nil { + return "", err + } + return AssignmentDir(root, workspace, checked) +} + +// pyEqual is Python == over decoded JSON values: numbers by value (a bool is 0 or 1), objects +// by key set. +func pyEqual(a, b any) bool { + number := func(v any) (float64, bool) { + switch n := v.(type) { + case bool: + if n { + return 1, true + } + return 0, true + case int64: + return float64(n), true + case int: + return float64(n), true + case float64: + return n, true + } + return 0, false + } + if x, ok := number(a); ok { + y, ok := number(b) + return ok && x == y && !math.IsNaN(x) + } + switch x := a.(type) { + case nil: + return b == nil + case string: + y, ok := b.(string) + return ok && x == y + case []any: + y, ok := b.([]any) + if !ok || len(x) != len(y) { + return false + } + for i := range x { + if !pyEqual(x[i], y[i]) { + return false + } + } + return true + case Obj: + y, ok := b.(Obj) + if !ok || len(x) != len(y) { + return false + } + for _, f := range x { + other, present := get(y, f.Key) + if !present || !pyEqual(f.Value, other) { + return false + } + } + return true + } + return false +} + +// publishOrCompare is _publish_or_compare: whether losing was a replay or a contradiction. +func publishOrCompare(target string, payload Obj, fields []string, root string, since []string) (string, error) { + outcome, err := Publish(target, payload, root) + if err != nil || outcome == Published { + return outcome, err + } + value, status := readFact(target) + existing, isRecord := value.(Obj) + if status != factPresent || !isRecord { + return Conflict, nil + } + for _, f := range fields { + if !pyEqual(fieldOf(existing, f), fieldOf(payload, f)) { + return Conflict, nil + } + } + for _, f := range since { + if stored, present := get(existing, f); present && !pyEqual(stored, fieldOf(payload, f)) { + return Conflict, nil + } + } + return Unchanged, nil +} + +// nextIndex is _next_index: the next free slot for a numbered fact, and whether it was read. +func nextIndex(directory, kind string) (int, bool) { + entries, readable := listing(filepath.Join(directory, kind), "*.json", "") + if !readable { + return 0, false + } + next := 0 + for _, entry := range entries { + name := filepath.Base(entry) + if strings.HasPrefix(name, ".") { + continue + } + if n, err := strconv.Atoi(stem(name)); err == nil && n >= 0 && strings.Trim(stem(name), "0123456789") == "" && n+1 > next { + next = n + 1 + } + } + return next, true +} + +// publishNumbered is _publish_numbered: the next numbered fact, re-listing on every loss. +func publishNumbered(directory, kind string, payload Obj, root string) (string, error) { + for range 64 { + index, readable := nextIndex(directory, kind) + if !readable { + return "", registrationError(RelationshipConflict, "the "+kind+" directory cannot be read, so no slot can be allocated in it") + } + outcome, err := Publish(filepath.Join(directory, kind, strconv.Itoa(index)+".json"), payload, root) + if err != nil { + return "", err + } + if outcome == Published { + return kind + "/" + strconv.Itoa(index), nil + } + } + return "", registrationError(RelationshipConflict, "could not allocate a free "+kind+" slot after 64 attempts") +} + +// IntentDeclaration is declare_intent's keyword arguments. +type IntentDeclaration struct { + Workspace, DispatchRequestID, IssueKey, DeclaredAt string + CriteriaSource, BaselineRevision, AuthorizedSettings, DBPath any +} + +// DeclareIntent is declare_intent: the fact that exists BEFORE the task does. The dispatch +// request id is stored only as its sha256. +func DeclareIntent(root string, d IntentDeclaration) (Obj, error) { + if !Named(d.DispatchRequestID) { + return nil, registrationError(UnboundGeneration, "an intent needs an exact dispatch request id") + } + assignment := AssignmentID(d.DispatchRequestID) + directory, err := assignmentDirectory(root, d.Workspace, assignment) + if err != nil { + return nil, err + } + workspace, err := resolved(d.Workspace) + if err != nil { + return nil, err + } + payload := Obj{{Key: "dispatchRequestIdHash", Value: assignment}, {Key: "issueKey", Value: d.IssueKey}, {Key: "workspace", Value: workspace}, + {Key: "criteriaSource", Value: d.CriteriaSource}, {Key: "baselineRevision", Value: d.BaselineRevision}, {Key: "authorizedSettings", Value: d.AuthorizedSettings}, {Key: "declaredAt", Value: d.DeclaredAt}} + if truthy(d.DBPath) { + payload = append(payload, F{Key: "dbPath", Value: pyStr(d.DBPath)}) + } + outcome, err := publishOrCompare(filepath.Join(directory, "intent.json"), payload, intentFields, root, nil) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "assignmentDir", Value: directory}, {Key: "outcome", Value: outcome}, {Key: "declaredAt", Value: d.DeclaredAt}}, nil +} + +// RecordAttempt is record_attempt: what the creation call returned (accepted, unknown, failed). +func RecordAttempt(root, workspace string, assignment any, outcome, at string, taskID any) (Obj, error) { + valid := false + for _, o := range AttemptOutcomes { + valid = valid || o == outcome + } + if !valid { + return nil, registrationError(UnknownGeneration, "an attempt outcome is one of "+strings.Join(AttemptOutcomes, ", ")+", not "+store.PyRepr(outcome)) + } + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + payload := Obj{{Key: "outcome", Value: outcome}, {Key: "at", Value: at}} + if taskID != nil { + payload = append(payload, F{Key: "taskId", Value: taskID}) + } + factID, err := publishNumbered(directory, "attempts", payload, root) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "factId", Value: factID}, {Key: "outcome", Value: outcome}}, nil +} + +// BindIdentity is bind: the intent bound to the real native task id, atomic and idempotent. +// A losing different identity is RECORDED as a conflict, never swallowed. +func BindIdentity(root, workspace string, assignment any, sessionID, taskID any, at string) (Obj, error) { + if !Named(sessionID) || !Named(taskID) { + return nil, registrationError(UnboundGeneration, "a bind needs an exact session id and task id; a record naming nothing binds nothing") + } + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + outcome, err := Publish(filepath.Join(directory, "bound.json"), Obj{{Key: "sessionId", Value: sessionID}, {Key: "taskId", Value: taskID}, {Key: "at", Value: at}}, root) + if err != nil { + return nil, err + } + if outcome == Published { + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "outcome", Value: Bound}, {Key: "sessionId", Value: sessionID}, {Key: "taskId", Value: taskID}}, nil + } + marker, unreadable := ReadAssignment(directory) + winnerValue, _ := get(marker, "bound") + winner, ok := winnerValue.(Obj) + if !ok { + labels := unreadable + if len(labels) == 0 { + labels = []string{"bound"} + } + return nil, registrationError(UnboundGeneration, "a bind already exists here and cannot be read: "+strings.Join(labels, ", ")) + } + if SameIdentity(fieldOf(winner, "sessionId"), sessionID) && SameIdentity(fieldOf(winner, "taskId"), taskID) { + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "outcome", Value: Unchanged}, {Key: "sessionId", Value: sessionID}, {Key: "taskId", Value: taskID}}, nil + } + factID, err := publishNumbered(directory, "conflicts", Obj{{Key: "attemptedSessionId", Value: sessionID}, {Key: "attemptedTaskId", Value: taskID}, {Key: "loserProcess", Value: strconv.Itoa(os.Getpid())}, {Key: "at", Value: at}}, root) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "outcome", Value: Conflict}, {Key: "factId", Value: factID}, {Key: "boundSessionId", Value: fieldOf(winner, "sessionId")}, {Key: "boundTaskId", Value: fieldOf(winner, "taskId")}}, nil +} + +// generationState is _generation_state: (state, generation) for one dispatch on a connection the +// caller owns; state "" when the read itself failed. +func generationState(ctx context.Context, conn *sql.Conn, relationshipID, dispatchRequestID string) (string, any) { + var opened, current any + err := conn.QueryRowContext(ctx, "SELECT g.execution_generation AS opened, r.execution_generation AS current FROM generations g JOIN relationships r ON r.relationship_id = g.relationship_id WHERE g.relationship_id = ? AND g.dispatch_request_id = ?", relationshipID, dispatchRequestID).Scan(&opened, ¤t) + if errors.Is(err, sql.ErrNoRows) { + return DispatchAbsent, nil + } + if err != nil { + return "", nil + } + if !pyEqual(opened, current) { + return DispatchStale, current + } + return DispatchCurrent, current +} + +// RegisterRelationship is register_relationship: which relay relationship this assignment was +// registered as, confirmed against the relay store and published under ONE hold on its write +// lock (store.RegistrationHold). dbPath nil is Python's None. +func RegisterRelationship(ctx context.Context, root, workspace string, assignment any, relationshipID, dispatchRequestID, at string, dbPath any) (Obj, error) { + if !Named(relationshipID) { + return nil, registrationError(UnregisteredRelationship, "a registration needs an exact relationship id") + } + if AssignmentID(dispatchRequestID) != assignment { + return nil, registrationError(RelationshipConflict, "relationship "+relationshipID+" was dispatched under a different request id, so it does not belong to assignment "+pyStr(assignment)) + } + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + refuseUnheld := func(unavailable string) error { + return registrationError(UnregisteredRelationship, "the relay store could not be held for this registration, so it cannot be confirmed that relationship "+relationshipID+" is still this assignment's at the moment the registration lands ("+unavailable+"); it is refused rather than published on the caller's word") + } + path, isPath := dbPath.(string) + if !isPath { + // Path(None) raises TypeError, which registration_hold answers as an unreadable path. + return nil, refuseUnheld("the relay store path " + pyReprValue(dbPath) + " could not be read as a path") + } + var result Obj + err = store.RegistrationHold(ctx, path, func(held *sql.Conn, unavailable string) error { + if held == nil { + return refuseUnheld(unavailable) + } + state, generation := generationState(ctx, held, relationshipID, dispatchRequestID) + switch state { + case "": + return registrationError(UnregisteredRelationship, "the relay store could not be read, so it cannot be confirmed that relationship "+relationshipID+" belongs to this assignment; registration is refused rather than taken on the caller's word") + case DispatchStale: + return registrationError(StaleGeneration, "this assignment's dispatch request id opened an earlier generation of relationship "+relationshipID+", which has since advanced, so registering it would attribute the current generation's receipts to a superseded assignment") + case DispatchAbsent: + return registrationError(RelationshipConflict, "the relay has no generation of relationship "+relationshipID+" opened under this assignment's dispatch request id, so it is not this assignment's relationship") + } + outcome, err := publishOrCompare(filepath.Join(directory, "relationship.json"), Obj{{Key: "relationshipId", Value: relationshipID}, {Key: "executionGeneration", Value: generation}, {Key: "at", Value: at}}, []string{"relationshipId"}, root, []string{"executionGeneration"}) + if err != nil { + return err + } + result = Obj{{Key: "assignmentId", Value: assignment}, {Key: "relationshipId", Value: relationshipID}, {Key: "executionGeneration", Value: generation}, {Key: "outcome", Value: outcome}} + return nil + }) + return result, err +} + +// PublishResolution is publish_resolution: adjudicate named evidence; resolutions accumulate. +func PublishResolution(root, workspace string, assignment, chosenTaskID, chosenSessionID, reason any, at string, adjudicated []Obj) (Obj, error) { + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + entries := make([]any, len(adjudicated)) + for i, entry := range adjudicated { + entries[i] = Obj{{Key: "factId", Value: fieldOf(entry, "factId")}, {Key: "digest", Value: fieldOf(entry, "digest")}} + } + factID, err := publishNumbered(directory, "resolutions", Obj{{Key: "chosenTaskId", Value: chosenTaskID}, {Key: "chosenSessionId", Value: chosenSessionID}, {Key: "reason", Value: reason}, {Key: "at", Value: at}, {Key: "adjudicated", Value: entries}}, root) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "factId", Value: factID}, {Key: "adjudicated", Value: int64(len(entries))}}, nil +} + +// PublishClaim is publish_claim: the child's own assertion that it is this assignment's session, +// refused where the dispatch it names does not hash to the assignment. +func PublishClaim(root, workspace string, assignment, sessionID any, dispatchRequestID string, firstTurnID any, at string) (Obj, error) { + session, err := checkedIdentity(sessionID, "session id") + if err != nil { + return nil, err + } + if AssignmentID(dispatchRequestID) != assignment { + return nil, registrationError(RelationshipConflict, "this claim names dispatch request id "+dispatchRequestID+", which does not hash to assignment "+pyStr(assignment)+", so it claims a different assignment") + } + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + outcome, err := publishOrCompare(filepath.Join(directory, "claims", session, claimFile), Obj{{Key: "dispatchRequestId", Value: dispatchRequestID}, {Key: "sessionId", Value: session}, {Key: "firstTurnId", Value: firstTurnID}, {Key: "at", Value: at}}, []string{"dispatchRequestId", "sessionId"}, root, nil) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "sessionId", Value: session}, {Key: "outcome", Value: outcome}}, nil +} + +// PublishDisposition is publish_disposition: what this turn declared, at the path its Stop +// identity derives, from the exhaustive vocabulary only. +func PublishDisposition(root, workspace string, assignment, sessionID, turnID any, outcome, at string) (Obj, error) { + session, err := checkedIdentity(sessionID, "session id") + if err != nil { + return nil, err + } + turn, err := checkedIdentity(turnID, "turn id") + if err != nil { + return nil, err + } + known := false + for _, o := range DispositionOutcomes { + known = known || o == outcome + } + if !known { + return nil, registrationError(OutcomeInconsistent, "a disposition outcome is one of "+strings.Join(DispositionOutcomes, ", ")+", not "+store.PyRepr(outcome)) + } + directory, err := assignmentDirectory(root, workspace, assignment) + if err != nil { + return nil, err + } + published, err := publishOrCompare(filepath.Join(directory, "dispositions", session, turn+".json"), Obj{{Key: "sessionId", Value: session}, {Key: "turnId", Value: turn}, {Key: "outcome", Value: outcome}, {Key: "at", Value: at}}, []string{"sessionId", "turnId", "outcome"}, root, nil) + if err != nil { + return nil, err + } + return Obj{{Key: "assignmentId", Value: assignment}, {Key: "sessionId", Value: session}, {Key: "turnId", Value: turn}, {Key: "outcome", Value: outcome}, {Key: "published", Value: published}}, nil +} diff --git a/internal/relay/delivery/intent_cli.go b/internal/relay/delivery/intent_cli.go new file mode 100644 index 00000000..b59a4b6a --- /dev/null +++ b/internal/relay/delivery/intent_cli.go @@ -0,0 +1,541 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/contract" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The eight marker commands of `codex-session-relay` (cli.py:3470-3760, parser :4960-5032): +// intent-declare/attempt/bind/register/claim/disposition/resolve/show, plus the store records +// intent-claim and intent-disposition mirror beside their marker facts (declarations.py). +// guard-evaluate, the ninth marker command, is todo 33's. + +func markerFlags(extra ...flagSpec) []flagSpec { + return append([]flagSpec{{"--marker-root", "store", false, nil, nil}, {"--workspace", "store", true, nil, nil}}, extra...) +} + +var intentCommands = map[string]commandSpec{ + "intent-declare": {flags: markerFlags(flagSpec{"--dispatch-request-id", "store", true, nil, nil}, flagSpec{"--issue", "store", true, nil, nil}, flagSpec{"--declared-at", "store", false, nil, nil}, + flagSpec{"--criteria-source", "store", false, nil, nil}, flagSpec{"--baseline-revision", "store", false, nil, nil}, flagSpec{"--settings", "store", false, nil, nil}, flagSpec{"--no-db-path", "true", false, nil, false}), + run: cmdIntentDeclare, exempt: func(a map[string]any) bool { return a["--no-db-path"] == true }}, + "intent-attempt": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--outcome", "store", true, AttemptOutcomes, nil}, flagSpec{"--task-id", "store", false, nil, nil}), + run: cmdIntentAttempt, exempt: always}, + "intent-bind": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--session", "store", true, nil, nil}, flagSpec{"--task-id", "store", true, nil, nil}), + run: cmdIntentBind, exempt: always}, + "intent-register": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--relationship", "store", true, nil, nil}, flagSpec{"--dispatch-request-id", "store", true, nil, nil}, flagSpec{"--db-path", "store", false, nil, nil}), + run: cmdIntentRegister, exempt: func(a map[string]any) bool { return truthy(a["--db-path"]) }}, + "intent-claim": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--session", "store", true, nil, nil}, flagSpec{"--dispatch-request-id", "store", true, nil, nil}, flagSpec{"--first-turn", "store", false, nil, nil}), + run: cmdIntentClaim, exempt: always}, + "intent-disposition": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--session", "store", true, nil, nil}, flagSpec{"--turn", "store", true, nil, nil}, flagSpec{"--outcome", "store", true, DispositionOutcomes, nil}), + run: cmdIntentDisposition, exempt: always}, + "intent-resolve": {flags: markerFlags(flagSpec{"--assignment", "store", true, nil, nil}, flagSpec{"--chosen-task", "store", true, nil, nil}, flagSpec{"--chosen-session", "store", true, nil, nil}, flagSpec{"--reason", "store", true, nil, nil}, flagSpec{"--adjudicate", "append", true, nil, nil}), + run: cmdIntentResolve, exempt: always}, + "intent-show": {flags: markerFlags(flagSpec{"--assignment", "store", false, nil, nil}, flagSpec{"--session", "store", false, nil, nil}, flagSpec{"--now", "store", false, nil, nil}), + run: cmdIntentShow, exempt: always}, +} + +// intentCommandNames is cli.py's add_parser order for the marker commands this package serves. +var intentCommandNames = []string{"intent-declare", "intent-attempt", "intent-bind", "intent-register", "intent-claim", "intent-disposition", "intent-resolve", "intent-show"} + +func init() { + for name, spec := range intentCommands { + deliveryCommands[name] = spec + } +} + +func always(map[string]any) bool { return true } + +// payloadExit is cli.PayloadExit: a whole answer printed with its own exit code. +type payloadExit struct { + payload Obj + code int +} + +func (p *payloadExit) Error() string { return pyStr(fieldOf(p.payload, "detail")) } +func (p *payloadExit) ExitPayload() (contract.OrderedObject, int) { return p.payload, p.code } + +// markerRoot is _marker_root: resolve_marker_root(args.marker_root).path. +func (c *cliRun) markerRoot() (string, error) { + selection, err := ResolveMarkerRoot(c.s("--marker-root")) + return selection.Path, err +} + +func (c *cliRun) dbPath() string { return c.state + "/relay.sqlite3" } + +func cmdIntentDeclare(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + declaredAt := c.s("--declared-at") + if declaredAt == "" { + declaredAt = c.clock.ISO() + } + var settings any + if raw := c.s("--settings"); raw != "" { + if settings, err = loads(raw); err != nil { + return nil, &hostError{"JSONDecodeError", err.Error()} + } + } + var db any + if c.args["--no-db-path"] != true { + db = c.dbPath() + } + return DeclareIntent(root, IntentDeclaration{Workspace: c.s("--workspace"), DispatchRequestID: c.s("--dispatch-request-id"), IssueKey: c.s("--issue"), DeclaredAt: declaredAt, + CriteriaSource: c.opt("--criteria-source"), BaselineRevision: c.opt("--baseline-revision"), AuthorizedSettings: settings, DBPath: db}) +} + +func cmdIntentAttempt(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + return RecordAttempt(root, c.s("--workspace"), c.s("--assignment"), c.s("--outcome"), c.clock.ISO(), c.opt("--task-id")) +} + +func cmdIntentBind(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + return BindIdentity(root, c.s("--workspace"), c.s("--assignment"), c.s("--session"), c.s("--task-id"), c.clock.ISO()) +} + +func cmdIntentRegister(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + db := c.s("--db-path") + if db == "" { + db = c.dbPath() + } + return RegisterRelationship(c.ctx, root, c.s("--workspace"), c.s("--assignment"), c.s("--relationship"), c.s("--dispatch-request-id"), c.clock.ISO(), db) +} + +// adjudicated is _adjudicated: every --adjudicate factId=digest, or a usage error. +func adjudicated(values []string) ([]Obj, error) { + entries := []Obj{} + for _, value := range values { + factID, digest, _ := strings.Cut(value, "=") + if factID == "" || digest == "" { + return nil, &usageError{"--adjudicate takes factId=digest, not " + store.PyRepr(value), contract.ExitUsage} + } + entries = append(entries, Obj{{Key: "factId", Value: factID}, {Key: "digest", Value: digest}}) + } + return entries, nil +} + +func cmdIntentResolve(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + entries, err := adjudicated(c.list("--adjudicate")) + if err != nil { + return nil, err + } + return PublishResolution(root, c.s("--workspace"), c.s("--assignment"), c.s("--chosen-task"), c.s("--chosen-session"), c.s("--reason"), c.clock.ISO(), entries) +} + +// orEmptyList is `value or []`. +func orEmptyList(v any) any { + if !truthy(v) { + return []any{} + } + return v +} + +func stringsAny(values []string) []any { + out := make([]any, len(values)) + for i, v := range values { + out[i] = v + } + return out +} + +func cmdIntentShow(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + workspace := c.s("--workspace") + var directory string + var facts Obj + var unreadable []string + if assignment := c.s("--assignment"); assignment != "" { + if directory, err = AssignmentDir(root, workspace, assignment); err != nil { + return nil, err + } + facts, unreadable = ReadAssignment(directory) + if _, has := get(facts, "intent"); len(unreadable) == 0 && !has { + return Obj{{Key: "markerRoot", Value: root}, {Key: "workspace", Value: workspace}, {Key: "managed", Value: false}, {Key: "assignmentId", Value: filepath.Base(directory)}, + {Key: "assignmentDir", Value: directory}, {Key: "unreadable", Value: []any{}}, {Key: "detail", Value: "no intent is published for this assignment"}}, nil + } + } else { + if directory, facts, unreadable, err = SelectAssignment(root, workspace, c.opt("--session")); err != nil { + return nil, err + } + if directory == "" { + return Obj{{Key: "markerRoot", Value: root}, {Key: "workspace", Value: workspace}, {Key: "managed", Value: false}, {Key: "unreadable", Value: stringsAny(unreadable)}}, nil + } + } + var malformed any + if len(facts) > 0 { + if m := Malformed(facts); m != "" { + malformed = m + } + } + payload := Obj{{Key: "markerRoot", Value: root}, {Key: "workspace", Value: workspace}, {Key: "managed", Value: true}, {Key: "assignmentId", Value: filepath.Base(directory)}, + {Key: "assignmentDir", Value: directory}, {Key: "unreadable", Value: stringsAny(unreadable)}, {Key: "malformed", Value: malformed}} + if malformed != nil || len(unreadable) > 0 { + return payload, nil + } + now := c.opt("--now") + if !truthy(now) { + now = c.clock.ISO() + } + return append(payload, + F{Key: "assignmentState", Value: DeriveAssignmentState(facts, now)}, + F{Key: "identityContested", Value: IdentityContested(facts)}, + F{Key: "intent", Value: fieldOf(facts, "intent")}, + F{Key: "bound", Value: fieldOf(facts, "bound")}, + F{Key: "relationship", Value: fieldOf(facts, "relationship")}, + F{Key: "attempts", Value: orEmptyList(fieldOf(facts, "attempts"))}, + F{Key: "claims", Value: orEmptyList(fieldOf(facts, "claims"))}, + F{Key: "conflicts", Value: orEmptyList(fieldOf(facts, "conflicts"))}, + F{Key: "resolutions", Value: orEmptyList(fieldOf(facts, "resolutions"))}), nil +} + +func cmdIntentClaim(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + workspace, assignment, dispatch := c.s("--workspace"), c.s("--assignment"), c.s("--dispatch-request-id") + published, err := PublishClaim(root, workspace, assignment, c.s("--session"), dispatch, c.opt("--first-turn"), c.clock.ISO()) + if err != nil { + return nil, err + } + directory, err := AssignmentDir(root, workspace, assignment) + if err != nil { + return nil, err + } + facts, unreadable := ReadAssignment(directory) + session := fieldOf(published, "sessionId") + var standing Obj + claims, _ := fieldOf(facts, "claims").([]any) + for _, item := range claims { + if claim, ok := item.(Obj); ok && Claimant(claim) == session { + standing = claim + break + } + } + intentFact, _ := fieldOf(facts, "intent").(Obj) + var record Obj + switch { + case len(unreadable) > 0: + sorted := slices.Clone(unreadable) + slices.Sort(sorted) + record = declNotRecorded("marker_unreadable", "the marker could not be read whole after the claim ("+strings.Join(sorted, ", ")+"), so this session records nothing about how it reports; running the claim again once the marker reads records it", nil) + case Malformed(facts) != "": + record = declNotRecorded("marker_malformed", "the marker's "+Malformed(facts)+" is not the shape a fact must be, so this session records nothing about how it reports", nil) + case fieldOf(published, "outcome") == Conflict || standing == nil || !pyEqual(fieldOf(standing, "dispatchRequestId"), dispatch): + record = declNotRecorded("claim_not_standing", "the marker does not stand on this claim, so this session records nothing about how it reports", nil) + default: + declared, _ := fieldOf(intentFact, "workspace").(string) + same := false + if Correlated(facts, session, assignment) && declared != "" { + left, errLeft := resolved(declared) + right, errRight := resolved(workspace) + if errLeft != nil || errRight != nil { + return nil, errors.Join(errLeft, errRight) + } + same = left == right + } + if !same { + record = declNotRecorded("claim_uncorrelated", "the claim does not correlate with the intent declared for this workspace, so the store derives nothing for this session", nil) + break + } + markerRoot, err := resolved(root) + if err != nil { + return nil, err + } + workspaceResolved, err := resolved(workspace) + if err != nil { + return nil, err + } + record = recordClaim(c.ctx, storeOf(facts), store.ReportingSessionsRow{AssignmentID: assignment, SessionID: pyStr(session), DispatchRequestID: dispatch, + MarkerRoot: markerRoot, Workspace: workspaceResolved, IssueKey: nullString(fieldOf(intentFact, "issueKey")), Capability: declarationsCapability, RecordedAt: c.clock.ISO()}) + } + return withStoreRecord(published, record) +} + +func cmdIntentDisposition(c *cliRun) (any, error) { + root, err := c.markerRoot() + if err != nil { + return nil, err + } + workspace, assignment := c.s("--workspace"), c.s("--assignment") + var before Obj + var unreadableBefore []string + directory, err := AssignmentDir(root, workspace, assignment) + if err == nil { + before, unreadableBefore = ReadAssignment(directory) + } else { + directory = "" + } + var published, record Obj + body := func(ctx context.Context, held *store.Store, heldPath string, problem Obj) error { + var err error + published, err = PublishDisposition(root, workspace, assignment, c.s("--session"), c.s("--turn"), c.s("--outcome"), c.clock.ISO()) + if err != nil { + return err + } + if directory == "" { + if directory, err = AssignmentDir(root, workspace, assignment); err != nil { + return err + } + } + facts, unreadable := ReadAssignment(directory) + standingValue, readable := ReadDisposition(directory, fieldOf(published, "sessionId"), fieldOf(published, "turnId")) + standing, _ := standingValue.(Obj) + switch { + case slices.Contains(unreadable, "intent") || slices.Contains(unreadableBefore, "intent") || !readable || !truthy(standingValue) || malformedDisposition(standingValue) != "": + record = declFailure("marker_unreadable", "the intent or the disposition could not be read back from the marker after it was published, so what the marker stands on is unknown", nil) + case !pyEqual(storeOf(facts), storeOf(before)): + record = declFailure("store_changed", "the intent named another store while this was being recorded, so the record was not written to either", nil) + case held == nil: + record = problem + default: + declaredAt := fieldOf(standing, "at") + if !truthy(declaredAt) { + declaredAt = "" + } + record = recordDisposition(ctx, held, heldPath, store.TurnDeclarationsRow{AssignmentID: assignment, SessionID: pyStr(fieldOf(published, "sessionId")), TurnID: pyStr(fieldOf(published, "turnId")), + Outcome: pyStr(fieldOf(standing, "outcome")), DeclaredAt: pyStr(declaredAt), RecordedAt: c.clock.ISO()}) + } + return nil + } + held, heldPath, problem := openDeclarationStore(c.ctx, storeOf(before)) + if held == nil { + if err := body(c.ctx, nil, "", problem); err != nil { + return nil, err + } + return withStoreRecord(published, record) + } + defer func() { _ = held.Close() }() + began := false + var bodyErr error + txErr := held.Transaction(c.ctx, func(ctx context.Context, _ *sql.Conn) error { + began = true + bodyErr = body(ctx, held, heldPath, nil) + return bodyErr + }) + switch { + case !began: + // Held: a store that cannot be locked is only the answer, and the publication goes ahead. + if err := body(c.ctx, nil, "", declFailure("store_locked", store.PythonSQLiteError(txErr), heldPath)); err != nil { + return nil, err + } + case bodyErr != nil: + return nil, bodyErr + case txErr != nil && str(record, "state") == declRecorded: + // Held.settled: a commit that failed undoes the record. + record = declFailure("store_write_failed", store.PythonSQLiteError(txErr), fieldOf(record, "store")) + } + return withStoreRecord(published, record) +} + +// malformedDisposition is intent.malformed_disposition. +func malformedDisposition(record any) string { + if record == nil { + return "" + } + o, ok := record.(Obj) + if !ok { + return "disposition" + } + for _, field := range []string{"sessionId", "turnId", "outcome"} { + if v, present := get(o, field); present { + if _, isString := v.(string); !isString { + return "disposition." + field + } + } + } + return "" +} + +// withStoreRecord is _with_store_record: the marker answer with the store record beside it; a +// failed record fails the command with the whole answer. +func withStoreRecord(published, record Obj) (any, error) { + payload := append(slices.Clone(published), F{Key: "storeRecord", Value: record}) + if str(record, "state") == declFailed { + return nil, &payloadExit{append(payload, F{Key: "detail", Value: "the marker fact was published and the relay store record was not: " + pyStr(fieldOf(record, "detail"))}), contract.ExitRefused} + } + return payload, nil +} + +// ---------------------------------------------------------------- declarations.py + +const ( + declarationsCapability = "declarations/1" + declRecorded = "recorded" + declNotRecordedState = "not_recorded" + declFailed = "failed" +) + +// storeOf is declarations.store_of: the store the coordinator recorded, or nil. +func storeOf(facts Obj) any { + declared, ok := fieldOf(facts, "intent").(Obj) + if !ok { + return nil + } + path, ok := fieldOf(declared, "dbPath").(string) + if !ok || strings.TrimSpace(path) == "" { + return nil + } + return path +} + +func declNotRecorded(reason, detail string, dbPath any) Obj { + return Obj{{Key: "recorded", Value: false}, {Key: "state", Value: declNotRecordedState}, {Key: "reason", Value: reason}, {Key: "store", Value: dbPath}, {Key: "detail", Value: detail}} +} + +func declFailure(reason, detail string, dbPath any) Obj { + return Obj{{Key: "recorded", Value: false}, {Key: "state", Value: declFailed}, {Key: "reason", Value: reason}, {Key: "store", Value: dbPath}, + {Key: "detail", Value: detail + ". The marker fact stands; running the same command again retries this record and changes nothing else"}} +} + +// pathlibString is str(Path(value)): repeated and trailing separators and "." parts dropped, +// ".." kept. +func pathlibString(value string) string { + if value == "" { + return "." + } + parts := []string{} + for _, part := range strings.Split(value, "/") { + if part != "" && part != "." { + parts = append(parts, part) + } + } + joined := strings.Join(parts, "/") + if strings.HasPrefix(value, "/") { + return "/" + joined + } + if joined == "" { + return "." + } + return joined +} + +// openDeclarationStore is declarations._open: the store to record in and its path, or nil and +// the answer saying why there is none. +func openDeclarationStore(ctx context.Context, dbPath any) (*store.Store, string, Obj) { + if dbPath == nil { + return nil, "", declNotRecorded("no_store_recorded", "the assignment's intent names no relay store, so there is none to record this in; the relay derives nothing from a store for this assignment", nil) + } + expanded, err := store.ExpandUser(dbPath.(string)) + if err != nil { + return nil, "", declFailure("store_unreadable", store.PythonOSError(err), dbPath) + } + path := pathlibString(expanded) + metadata, err := os.Stat(path) + switch { + case errors.Is(err, os.ErrNotExist): + return nil, path, declNotRecorded("store_absent", "the relay store the intent names does not exist, so nothing can be derived from it either; nothing was created", path) + case err != nil: + return nil, path, declFailure("store_unreadable", store.PythonOSError(err), path) + case !metadata.Mode().IsRegular(): + return nil, path, declFailure("store_not_a_file", "the path the intent names is not a regular file", path) + } + s, err := store.Open(ctx, path, "") + if err != nil { + return nil, path, declFailure("store_unopenable", store.PythonSQLiteError(err), path) + } + return s, path, nil +} + +func declRecordedAnswer(path string) Obj { + return Obj{{Key: "recorded", Value: true}, {Key: "state", Value: declRecorded}, {Key: "reason", Value: nil}, {Key: "store", Value: path}, {Key: "detail", Value: nil}} +} + +func declUnchangedAnswer(path string) Obj { + return Obj{{Key: "recorded", Value: false}, {Key: "state", Value: Unchanged}, {Key: "reason", Value: nil}, {Key: "store", Value: path}, {Key: "detail", Value: "already recorded, identically"}} +} + +func declConflictAnswer(path string, existing Obj) Obj { + return Obj{{Key: "recorded", Value: false}, {Key: "state", Value: Conflict}, {Key: "reason", Value: "store_disagrees"}, {Key: "store", Value: path}, + {Key: "detail", Value: "this store already holds a different record for it, and the first one stands, as it does in the marker: " + pyReprValue(existing)}} +} + +func nullString(v any) sql.NullString { + s, ok := v.(string) + return sql.NullString{String: s, Valid: ok} +} + +func nullValue(n sql.NullString) any { + if !n.Valid { + return nil + } + return n.String +} + +// recordClaim is declarations.record_claim: create-once in a write of its own. +func recordClaim(ctx context.Context, dbPath any, row store.ReportingSessionsRow) Obj { + s, path, problem := openDeclarationStore(ctx, dbPath) + if s == nil { + return problem + } + defer func() { _ = s.Close() }() + var answer Obj + err := s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + existing, err := s.ReportingSession(ctx, row.AssignmentID, row.SessionID) + if errors.Is(err, sql.ErrNoRows) { + if err := s.RecordReportingSession(ctx, row); err != nil { + return err + } + answer = declRecordedAnswer(path) + return nil + } + if err != nil { + return err + } + if existing.DispatchRequestID == row.DispatchRequestID && existing.MarkerRoot == row.MarkerRoot && existing.Workspace == row.Workspace && existing.IssueKey == row.IssueKey && existing.Capability == row.Capability { + answer = declUnchangedAnswer(path) + return nil + } + answer = declConflictAnswer(path, Obj{{Key: "dispatch_request_id", Value: existing.DispatchRequestID}, {Key: "marker_root", Value: existing.MarkerRoot}, + {Key: "workspace", Value: existing.Workspace}, {Key: "issue_key", Value: nullValue(existing.IssueKey)}, {Key: "capability", Value: existing.Capability}}) + return nil + }) + if err != nil { + return declFailure("store_write_failed", store.PythonSQLiteError(err), path) + } + return answer +} + +// recordDisposition is Held.disposition: the declared outcome, create-once, on the held write. +func recordDisposition(ctx context.Context, held *store.Store, path string, row store.TurnDeclarationsRow) Obj { + existing, err := held.TurnDeclaration(ctx, row.AssignmentID, row.SessionID, row.TurnID) + switch { + case errors.Is(err, sql.ErrNoRows): + if err := held.RecordTurnDeclaration(ctx, row); err != nil { + return declFailure("store_write_failed", store.PythonSQLiteError(err), path) + } + return declRecordedAnswer(path) + case err != nil: + return declFailure("store_write_failed", store.PythonSQLiteError(err), path) + case existing.Outcome == row.Outcome: + return declUnchangedAnswer(path) + } + return declConflictAnswer(path, Obj{{Key: "outcome", Value: existing.Outcome}}) +} diff --git a/internal/relay/delivery/intent_cli_test.go b/internal/relay/delivery/intent_cli_test.go new file mode 100644 index 00000000..10379cc4 --- /dev/null +++ b/internal/relay/delivery/intent_cli_test.go @@ -0,0 +1,130 @@ +package delivery + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// The eight intent-* marker commands through the real processes: Python's codex-session-relay +// and the built `crw relay`, each with its own home, marker root and seeded store, over one +// shared workspace path (the workspace key hashes it). Every stdout is compared whole after the +// wall-clock stamps, each side's home and each side's pid (loserProcess) become tokens; exit +// codes are compared exactly. + +var loserProcess = regexp.MustCompile(`"loserProcess": "\d+"`) + +func TestCLI_every_intent_command_answers_byte_for_byte_like_python(t *testing.T) { + work := filepath.Join(t.TempDir(), "work") + py, gosd := newSide(t, true, work), newSide(t, false, work) + rid := regexp.MustCompile(`rel-[0-9a-f]{16}`).FindString(sqliteDump(t, py, "SELECT relationship_id FROM relationships")) + if rid == "" || !strings.Contains(sqliteDump(t, gosd, "SELECT relationship_id FROM relationships"), rid) { + t.Fatal("both sides are seeded with the same relationship") + } + a1, a2 := AssignmentID("dispatch-1"), AssignmentID("dispatch-2") + cases := [][]string{ + {"intent-show", "--workspace", ""}, + {"intent-show", "--workspace", "", "--assignment", a1}, + {"intent-show", "--workspace", "", "--assignment", "../escape"}, + {"intent-declare", "--workspace", "", "--dispatch-request-id", "dispatch-1", "--issue", "REL-1", "--declared-at", "2026-01-01T00:00:00+00:00", "--criteria-source", "doc-a", "--settings", `{"sandbox": "workspace-write", "n": [1, 2.5]}`}, + {"intent-declare", "--workspace", "", "--dispatch-request-id", "dispatch-1", "--issue", "REL-1", "--declared-at", "2026-01-01T00:05:00+00:00", "--criteria-source", "doc-a", "--settings", `{"sandbox": "workspace-write", "n": [1, 2.5]}`}, + {"intent-declare", "--workspace", "", "--dispatch-request-id", "dispatch-1", "--issue", "REL-9"}, + {"intent-declare", "--workspace", "", "--dispatch-request-id", " ", "--issue", "REL-1"}, + {"intent-declare", "--workspace", "", "--dispatch-request-id", "dispatch-2", "--issue", "REL-2", "--no-db-path", "--declared-at", "2026-01-01T00:00:00+00:00"}, + {"intent-show", "--workspace", "", "--now", "2026-01-01T00:10:00+00:00"}, + {"intent-attempt", "--workspace", "", "--assignment", a1, "--outcome", "accepted", "--task-id", "01child-task"}, + {"intent-attempt", "--workspace", "", "--assignment", a1, "--outcome", "maybe"}, + {"intent-attempt", "--workspace", "", "--assignment", "not-hex", "--outcome", "failed"}, + {"intent-bind", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--task-id", "01child-task"}, + {"intent-bind", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--task-id", "01child-task"}, + {"intent-bind", "--workspace", "", "--assignment", a1, "--session", "01other", "--task-id", "01other"}, + {"intent-bind", "--workspace", "", "--assignment", a1, "--session", " ", "--task-id", "01other"}, + {"intent-register", "--workspace", "", "--assignment", a1, "--relationship", rid, "--dispatch-request-id", "some-other-dispatch"}, + {"intent-register", "--workspace", "", "--assignment", a1, "--relationship", rid, "--dispatch-request-id", "dispatch-1", "--db-path", "/no-such-store.sqlite3"}, + {"intent-register", "--workspace", "", "--assignment", a1, "--relationship", "rel-0123456789abcdef", "--dispatch-request-id", "dispatch-1"}, + {"intent-register", "--workspace", "", "--assignment", a1, "--relationship", rid, "--dispatch-request-id", "dispatch-1"}, + {"intent-register", "--workspace", "", "--assignment", a1, "--relationship", rid, "--dispatch-request-id", "dispatch-1", "--db-path", "/relay.sqlite3"}, + {"intent-claim", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--dispatch-request-id", "dispatch-1", "--first-turn", "turn-1"}, + {"intent-claim", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--dispatch-request-id", "dispatch-1", "--first-turn", "turn-1"}, + {"intent-claim", "--workspace", "", "--assignment", a1, "--session", "../escape", "--dispatch-request-id", "dispatch-1"}, + {"intent-claim", "--workspace", "", "--assignment", a1, "--session", "01second", "--dispatch-request-id", "dispatch-9"}, + {"intent-claim", "--workspace", "", "--assignment", a2, "--session", "01child-two", "--dispatch-request-id", "dispatch-2"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "turn-1", "--outcome", "ready_for_review"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "turn-1", "--outcome", "ready_for_review"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "turn-1", "--outcome", "failed"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "turn-2", "--outcome", "done"}, + // A store already holding another outcome for this turn: store_disagrees, first one stands. + {"!sql", "INSERT INTO turn_declarations VALUES ('" + a1 + "', '01child-task', 'turn-3', 'failed', 'x', 'y')"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "turn-3", "--outcome", "ready_for_review"}, + {"intent-disposition", "--workspace", "", "--assignment", a2, "--session", "01child-two", "--turn", "turn-1", "--outcome", "in_progress"}, + {"intent-disposition", "--workspace", "", "--assignment", a1, "--session", "01child-task", "--turn", "..", "--outcome", "failed"}, + {"intent-resolve", "--workspace", "", "--assignment", a1, "--chosen-task", "01child-task", "--chosen-session", "01child-task", "--reason", "r", "--adjudicate", "conflicts/0"}, + {"intent-resolve", "--workspace", "", "--assignment", a1, "--chosen-task", "01child-task", "--chosen-session", "01child-task", "--reason", "r", "--adjudicate", "conflicts/0=abc", "--adjudicate", "claims/x/claim.json=def"}, + {"intent-show", "--workspace", "", "--assignment", a1, "--now", "2026-01-01T00:10:00+00:00"}, + {"intent-show", "--workspace", "", "--session", "01child-task", "--now", "2026-01-01T01:00:00+00:00"}, + {"intent-show", "--workspace", "", "--session", "01child-two", "--now", "2026-01-01T00:10:00+00:00"}, + {"intent-show", "--workspace", "", "--marker-root", "/elsewhere"}, + {"intent-show"}, + } + for i, args := range cases { + if args[0] == "!sql" { + for _, side := range []*cliSide{py, gosd} { + cmd := execUV(side, "python", "-c", "import sqlite3,sys;c=sqlite3.connect(sys.argv[1]);c.execute(sys.argv[2]);c.commit()", filepath.Join(side.state, "relay.sqlite3"), args[1]) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v: %s", err, out) + } + } + continue + } + expand := func(s *cliSide) []string { + out := make([]string, len(args)) + for j, a := range args { + a = strings.ReplaceAll(a, "", s.work) + a = strings.ReplaceAll(a, "", s.state) + out[j] = strings.ReplaceAll(a, "", s.home) + } + if !strings.Contains(strings.Join(out, " "), "--marker-root") && len(out) > 1 { + out = append(out, "--marker-root", filepath.Join(s.home, "markers")) + } + return out + } + pout, pcode := py.run(expand(py)...) + gout, gcode := gosd.run(expand(gosd)...) + pn := loserProcess.ReplaceAllString(py.normal(pout), `"loserProcess": ""`) + gn := loserProcess.ReplaceAllString(gosd.normal(gout), `"loserProcess": ""`) + if os.Getenv("CRW_SHOW") != "" { + t.Logf("case %d exit %d\n%s", i, pcode, pn) + } + if pcode != gcode || pn != gn { + t.Errorf("case %d %v: exit python %d go %d\npython:\n%s\ngo:\n%s", i, args, pcode, gcode, pn, gn) + } + } + // The mirrored store records: the same rows on both sides. + for _, table := range []string{"reporting_sessions", "turn_declarations"} { + query := "SELECT * FROM " + table + " ORDER BY 1, 2, 3" + pr, gr := py.normal(sqliteDump(t, py, query)), gosd.normal(sqliteDump(t, gosd, query)) + pr, gr = stamp.ReplaceAllString(pr, ""), stamp.ReplaceAllString(gr, "") + if pr != gr || !strings.Contains(pr, "01child-task") { + t.Errorf("%s differs\npython: %s\ngo: %s", table, pr, gr) + } + } +} + +func execUV(s *cliSide, args ...string) *exec.Cmd { + cmd := exec.Command("uv", append([]string{"run", "--no-sync"}, args...)...) + cmd.Dir = filepath.Join(repoRoot(s.t), "packages", "codex-session-relay") + cmd.Env = s.env + return cmd +} + +func sqliteDump(t *testing.T, s *cliSide, query string) string { + t.Helper() + cmd := execUV(s, "python", "-c", "import sqlite3,sys,json;print(json.dumps([list(r) for r in sqlite3.connect(sys.argv[1]).execute(sys.argv[2])]))", filepath.Join(s.state, "relay.sqlite3"), query) + out, err := cmd.Output() + mustDo(t, err) + return string(out) +} diff --git a/internal/relay/delivery/intent_test.go b/internal/relay/delivery/intent_test.go new file mode 100644 index 00000000..2b007f1e --- /dev/null +++ b/internal/relay/delivery/intent_test.go @@ -0,0 +1,513 @@ +package delivery + +import ( + "strings" + "testing" +) + +// test_intent.py properties INT-1..INT-23. Each runs one list of operations through the real +// intent.py and through this package over the same tree, and every answer - returned records, +// refusal reason AND detail, derived states - is compared whole (testdata/markerops.py). + +const ( + intentT0 = "2026-01-01T00:00:00+00:00" + intentT5 = "2026-01-01T00:05:00+00:00" + intentT40 = "2026-01-01T00:40:00+00:00" + session1 = "01child-session" + task1 = "01child-task" + rel1 = "rel-0123456789abcdef" + relOther = "rel-ffffffffffffffff" +) + +func declareOp() markerOp { return markerOp{"op": "declare"} } +func bindOp() markerOp { return markerOp{"op": "bind", "session": session1, "task": task1} } +func openOp() markerOp { return markerOp{"op": "open_generation", "relationship_id": rel1} } +func registerOp() markerOp { + return markerOp{"op": "register", "relationship_id": rel1} +} + +func TestINT01_the_derived_state_follows_the_published_facts(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), markerOp{"op": "state"}, + markerOp{"op": "attempt", "outcome": "accepted", "task_id": task1}, markerOp{"op": "state"}, + bindOp(), markerOp{"op": "state"}, + openOp(), registerOp(), markerOp{"op": "state"}, + // T3: a lost creation response is unknown; a failed one is neither. + markerOp{"op": "declare", "dispatch": "d-unknown"}, markerOp{"op": "attempt", "dispatch": "d-unknown", "outcome": "unknown"}, markerOp{"op": "state", "dispatch": "d-unknown"}, + markerOp{"op": "declare", "dispatch": "d-failed"}, markerOp{"op": "attempt", "dispatch": "d-failed", "outcome": "failed"}, markerOp{"op": "state", "dispatch": "d-failed"}, + ) + for i, want := range map[int]string{1: IntentDeclared, 3: CreationAccepted, 5: IdentityBound, 8: RelationshipRegistered, 11: CreationUnknown, 14: IntentDeclared} { + if ok(t, answers[i]) != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT02_two_accepted_task_ids_are_ambiguous_and_ambiguity_outranks_expiry(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "attempt", "outcome": "accepted", "task_id": "task-a"}, + markerOp{"op": "attempt", "outcome": "accepted", "task_id": "task-b"}, + markerOp{"op": "state", "now": intentT5}, + markerOp{"op": "state", "now": intentT40}, + ) + if ok(t, answers[3]) != AmbiguousIdentity || ok(t, answers[4]) != AmbiguousIdentity { + t.Fatalf("states %v", answers) + } +} + +func TestINT03_expiry_is_anchored_on_declaredat_only(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), markerOp{"op": "attempt", "outcome": "accepted", "task_id": task1, "at": intentT40}, markerOp{"op": "state", "now": intentT40}, + markerOp{"op": "declare", "dispatch": "d-unknown"}, markerOp{"op": "attempt", "dispatch": "d-unknown", "outcome": "unknown"}, markerOp{"op": "state", "dispatch": "d-unknown", "now": intentT40}, + // Offsets and a naive stamp are compared as instants. + markerOp{"op": "declare", "dispatch": "d-offset", "declared_at": "2026-01-01T02:00:00+02:00"}, markerOp{"op": "state", "dispatch": "d-offset", "now": "2026-01-01T00:31:00"}, + markerOp{"op": "state", "dispatch": "d-offset", "now": "2026-01-01T00:29:00Z"}, + ) + for i, want := range map[int]string{2: IntentExpired, 5: IntentExpired, 7: IntentExpired, 8: IntentDeclared} { + if ok(t, answers[i]) != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT04_a_late_fact_cannot_move_the_state_backwards(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), markerOp{"op": "attempt", "outcome": "accepted", "task_id": task1}, bindOp(), openOp(), registerOp(), + markerOp{"op": "attempt", "outcome": "accepted", "task_id": task1, "at": intentT5}, + markerOp{"op": "state"}, + ) + if ok(t, answers[6]) != RelationshipRegistered { + t.Fatalf("state %v", answers[6]) + } +} + +func TestINT05_bind_is_create_once_and_a_loser_is_recorded(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), bindOp(), markerOp{"op": "facts"}, + markerOp{"op": "bind", "session": "other-session", "task": "other-task"}, + markerOp{"op": "bind", "session": "second", "task": "second-task"}, + markerOp{"op": "facts"}, + ) + if ok(t, answers[1]).(map[string]any)["outcome"] != Bound || ok(t, answers[2]).(map[string]any)["outcome"] != Unchanged { + t.Fatalf("binds %v", answers) + } + losing := ok(t, answers[4]).(map[string]any) + if losing["outcome"] != Conflict || losing["boundSessionId"] != session1 { + t.Fatalf("losing %v", losing) + } + facts := ok(t, answers[6]).(map[string]any)["facts"].(map[string]any) + if facts["bound"].(map[string]any)["sessionId"] != session1 || len(facts["conflicts"].([]any)) != 2 || facts["conflicts"].([]any)[0].(map[string]any)["attemptedSessionId"] != "other-session" { + t.Fatalf("facts %v", facts) + } +} + +func TestINT06_a_bind_naming_nothing_and_a_blank_dispatch_are_refused(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "bind", "session": "", "task": task1}, + markerOp{"op": "declare", "dispatch": " "}, + ) + if reasonOf(t, answers[1]) != UnboundGeneration || reasonOf(t, answers[2]) != UnboundGeneration { + t.Fatalf("refusals %v", answers) + } +} + +func TestINT07_intent_declaration_is_create_once_on_every_semantic_field(t *testing.T) { + base := func(field string, value any) markerOp { + op := markerOp{"op": "declare", "criteria_source": "doc-a", "baseline_revision": "rev-1", "issue_key": "REL-1", "db_path": "/first/relay.sqlite3"} + if field != "" { + op[field] = value + } + return op + } + answers := sameOps(t, nil, + base("", nil), + base("db_path", "/corrected/relay.sqlite3"), + base("criteria_source", "doc-b"), + base("baseline_revision", "rev-2"), + base("issue_key", "REL-9"), + base("", nil), + markerOp{"op": "facts"}, + ) + if ok(t, answers[0]).(map[string]any)["outcome"] != Published || ok(t, answers[5]).(map[string]any)["outcome"] != Unchanged { + t.Fatalf("outcomes %v", answers) + } + for i := 1; i <= 4; i++ { + if ok(t, answers[i]).(map[string]any)["outcome"] != Conflict { + t.Fatalf("answer %d %v", i, answers[i]) + } + } + if ok(t, answers[6]).(map[string]any)["facts"].(map[string]any)["intent"].(map[string]any)["dbPath"] != "/first/relay.sqlite3" { + t.Fatal("the stored intent was rewritten") + } +} + +func TestINT08_the_dispatch_request_id_is_stored_only_as_its_hash(t *testing.T) { + answers := sameOps(t, nil, declareOp(), markerOp{"op": "facts"}) + intent := ok(t, answers[1]).(map[string]any)["facts"].(map[string]any)["intent"].(map[string]any) + if strings.Contains(renderPlain(intent), "dispatch-request-1") || intent["dispatchRequestIdHash"] != AssignmentID("dispatch-request-1") { + t.Fatalf("intent %v", intent) + } +} + +func renderPlain(v any) string { return strings.ReplaceAll(pyReprValue(fromJSON(v)), " ", "") } + +func TestINT09_registration_is_confirmed_against_the_relay_store(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), + // Another dispatch's relationship. + markerOp{"op": "open_generation", "relationship_id": rel1}, markerOp{"op": "register", "relationship_id": rel1, "register_dispatch": "a-different-dispatch"}, + // An unrelated relationship carrying the right dispatch id. + markerOp{"op": "register", "relationship_id": relOther}, + // A dispatch whose generation has advanced. + markerOp{"op": "open_generation", "relationship_id": "rel-stale", "generation": 1, "current": 2}, markerOp{"op": "register", "relationship_id": "rel-stale"}, + markerOp{"op": "facts"}, + // The current generation registers. + markerOp{"op": "register", "relationship_id": rel1}, markerOp{"op": "facts"}, + ) + for i, want := range map[int]string{3: RelationshipConflict, 4: RelationshipConflict, 6: StaleGeneration} { + if reasonOf(t, answers[i]) != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } + if _, published := ok(t, answers[7]).(map[string]any)["facts"].(map[string]any)["relationship"]; published { + t.Fatal("a refusal published") + } + if ok(t, answers[8]).(map[string]any)["outcome"] != Published { + t.Fatalf("current %v", answers[8]) + } +} + +func TestINT10_an_unreadable_missing_or_held_store_refuses_registration(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), + markerOp{"op": "register", "relationship_id": rel1, "db_path": "/no-such-store.sqlite3"}, + markerOp{"op": "exists", "path": "no-such-store.sqlite3"}, + markerOp{"op": "register", "relationship_id": rel1, "db_path": "/state/no-such-store.sqlite3"}, + markerOp{"op": "exists", "path": "state/no-such-store.sqlite3"}, + openOp(), markerOp{"op": "hold_begin"}, + registerOp(), + markerOp{"op": "hold_end"}, + markerOp{"op": "facts"}, + ) + for _, i := range []int{2, 4, 8} { + if reasonOf(t, answers[i]) != UnregisteredRelationship { + t.Fatalf("answer %d %v", i, answers[i]) + } + } + if ok(t, answers[3]) != false || ok(t, answers[5]) != false { + t.Fatal("the refusal created the store it could not find") + } + if detail := answers[8].(map[string]any)["detail"].(string); !strings.Contains(detail, "write lock could not be taken") { + t.Fatalf("detail %q", detail) + } + if _, published := ok(t, answers[10]).(map[string]any)["facts"].(map[string]any)["relationship"]; published { + t.Fatal("published without the hold") + } +} + +func TestINT11_registration_is_create_once_and_names_a_contradiction(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), openOp(), registerOp(), registerOp(), + markerOp{"op": "open_generation", "relationship_id": relOther}, + markerOp{"op": "register", "relationship_id": relOther}, + markerOp{"op": "facts"}, + ) + if ok(t, answers[3]).(map[string]any)["outcome"] != Published || ok(t, answers[4]).(map[string]any)["outcome"] != Unchanged || ok(t, answers[6]).(map[string]any)["outcome"] != Conflict { + t.Fatalf("outcomes %v", answers) + } + if ok(t, answers[7]).(map[string]any)["facts"].(map[string]any)["relationship"].(map[string]any)["relationshipId"] != rel1 { + t.Fatal("the stored registration moved") + } +} + +func TestINT12_a_malformed_assignment_id_is_refused(t *testing.T) { + var ops []markerOp + ops = append(ops, declareOp()) + for _, bad := range []string{"../escape", "not-hex", "", strings.Repeat("A", 64)} { + ops = append(ops, markerOp{"op": "attempt", "assignment": bad, "outcome": "accepted", "task_id": task1}) + } + answers := sameOps(t, nil, ops...) + for i := 1; i < len(answers); i++ { + if reasonOf(t, answers[i]) != UnknownGeneration { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT13_the_registration_fact_records_its_generation(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), openOp(), + registerOp(), registerOp(), markerOp{"op": "facts"}, + // A legacy fact without the field replays unchanged and is not rewritten. + markerOp{"op": "declare", "dispatch": "legacy"}, + markerOp{"op": "open_generation", "relationship_id": "rel-legacy", "generation_dispatch": "legacy"}, + markerOp{"op": "publish", "dispatch": "legacy", "path": "relationship.json", "confined": true, "payload": map[string]any{"relationshipId": "rel-legacy", "at": intentT0}}, + markerOp{"op": "register", "dispatch": "legacy", "register_dispatch": "legacy", "relationship_id": "rel-legacy"}, + markerOp{"op": "facts", "dispatch": "legacy"}, + markerOp{"op": "open_generation", "relationship_id": relOther, "generation_dispatch": "legacy"}, + markerOp{"op": "register", "dispatch": "legacy", "register_dispatch": "legacy", "relationship_id": relOther}, + // A stored fact naming another generation is a contradiction. + markerOp{"op": "declare", "dispatch": "other-gen"}, + markerOp{"op": "open_generation", "relationship_id": "rel-other-gen", "generation_dispatch": "other-gen"}, + markerOp{"op": "publish", "dispatch": "other-gen", "path": "relationship.json", "confined": true, "payload": map[string]any{"relationshipId": "rel-other-gen", "executionGeneration": 2, "at": intentT0}}, + markerOp{"op": "register", "dispatch": "other-gen", "register_dispatch": "other-gen", "relationship_id": "rel-other-gen"}, + markerOp{"op": "facts", "dispatch": "other-gen"}, + ) + first := ok(t, answers[3]).(map[string]any) + if first["outcome"] != Published || first["executionGeneration"] != float64(1) || ok(t, answers[4]).(map[string]any)["outcome"] != Unchanged { + t.Fatalf("fresh %v %v", first, answers[4]) + } + if ok(t, answers[9]).(map[string]any)["outcome"] != Unchanged || ok(t, answers[12]).(map[string]any)["outcome"] != Conflict { + t.Fatalf("legacy %v %v", answers[9], answers[12]) + } + if _, has := ok(t, answers[10]).(map[string]any)["facts"].(map[string]any)["relationship"].(map[string]any)["executionGeneration"]; has { + t.Fatal("legacy fact rewritten") + } + conflict := ok(t, answers[16]).(map[string]any) + stored := ok(t, answers[17]).(map[string]any)["facts"].(map[string]any)["relationship"].(map[string]any) + if conflict["outcome"] != Conflict || conflict["executionGeneration"] != float64(1) || stored["executionGeneration"] != float64(2) { + t.Fatalf("other generation %v %v", conflict, stored) + } +} + +func TestINT14_the_registration_hold_records_nothing_in_the_relay(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), bindOp(), openOp(), + markerOp{"op": "relay_tables"}, + registerOp(), + markerOp{"op": "relay_tables"}, + ) + if ok(t, answers[4]).(map[string]any)["outcome"] != Published { + t.Fatalf("register %v", answers[4]) + } + requireSameJSON(t, "relay tables across the hold", ok(t, answers[5]), ok(t, answers[3])) +} + +func TestINT15_the_claimant_comes_from_the_path_and_the_body_must_agree(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "claim", "session": session1}, + markerOp{"op": "claimant", "fact": "claims/" + session1 + "/claim.json"}, + markerOp{"op": "publish", "path": "claims/impostor/claim.json", "payload": map[string]any{"dispatchRequestId": "dispatch-request-1", "sessionId": session1, "at": intentT0}}, + markerOp{"op": "claimant", "fact": "claims/impostor/claim.json"}, + markerOp{"op": "declare", "dispatch": "path-only"}, + markerOp{"op": "publish", "dispatch": "path-only", "path": "claims/" + session1 + "/claim.json", "payload": map[string]any{"at": intentT0}}, + markerOp{"op": "facts", "dispatch": "path-only"}, + ) + if ok(t, answers[2]) != session1 || ok(t, answers[4]) != nil { + t.Fatalf("claimants %v %v", answers[2], answers[4]) + } + claims := ok(t, answers[7]).(map[string]any)["facts"].(map[string]any)["claims"].([]any) + if len(claims) != 1 { + t.Fatalf("a path-only claim still exists: %v", claims) + } + if Claimant(fromJSON(claims[0]).(Obj)) != nil { + t.Fatal("a path-only claim owns nothing") + } +} + +func TestINT16_a_claim_must_correlate_with_the_intents_dispatch(t *testing.T) { + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "claim", "session": session1, "claim_dispatch": "not-the-dispatch-id"}, + markerOp{"op": "facts"}, + markerOp{"op": "publish", "path": "claims/" + session1 + "/claim.json", "payload": map[string]any{"dispatchRequestId": "not-the-dispatch-id", "sessionId": session1, "at": intentT0}}, + markerOp{"op": "correlated", "session": session1}, + markerOp{"op": "publish", "path": "claims/second/claim.json", "payload": map[string]any{"dispatchRequestId": "dispatch-request-1", "sessionId": "second", "at": intentT0}}, + markerOp{"op": "correlated", "session": "second"}, + ) + if reasonOf(t, answers[1]) != RelationshipConflict || !strings.Contains(answers[1].(map[string]any)["detail"].(string), "different assignment") { + t.Fatalf("refusal %v", answers[1]) + } + if claims := ok(t, answers[2]).(map[string]any)["facts"].(map[string]any)["claims"].([]any); len(claims) != 0 { + t.Fatal("the refused claim was written") + } + if ok(t, answers[4]) != false || ok(t, answers[6]) != true { + t.Fatalf("correlation %v %v", answers[4], answers[6]) + } +} + +func TestINT17_a_contest_clears_only_by_the_bound_identity_and_a_matching_digest(t *testing.T) { + second := "claims/second/claim.json" + answers := sameOps(t, nil, + declareOp(), markerOp{"op": "claim", "session": session1}, bindOp(), + markerOp{"op": "contested"}, + markerOp{"op": "publish", "path": second, "payload": map[string]any{"dispatchRequestId": "dispatch-request-1", "sessionId": "second", "at": intentT5}}, + markerOp{"op": "contested"}, + markerOp{"op": "resolution", "task": task1, "session": session1, "facts": []any{second}, "digest": "zero"}, + markerOp{"op": "covered", "fact": second}, + markerOp{"op": "contested"}, + markerOp{"op": "resolution", "task": task1, "session": "second", "facts": []any{second}}, + markerOp{"op": "contested"}, + markerOp{"op": "resolution", "task": task1, "session": session1, "facts": []any{second}}, + markerOp{"op": "covered", "fact": second}, + markerOp{"op": "contested"}, + // T16: a competing claim leaving its session blank still competes. + markerOp{"op": "declare", "dispatch": "blank"}, markerOp{"op": "claim", "dispatch": "blank", "claim_dispatch": "blank", "session": session1}, + markerOp{"op": "bind", "dispatch": "blank", "session": session1, "task": task1}, + markerOp{"op": "publish", "dispatch": "blank", "path": "claims/blank/claim.json", "payload": map[string]any{"dispatchRequestId": "blank", "sessionId": "", "at": intentT5}}, + markerOp{"op": "contested", "dispatch": "blank"}, + ) + for i, want := range map[int]bool{3: false, 5: true, 7: false, 8: true, 10: true, 12: true, 13: false, 18: true} { + if ok(t, answers[i]) != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT18_pre_bind_ambiguity_survives_disagreeing_or_unaccepted_resolutions(t *testing.T) { + everything := []any{"attempts/0", "attempts/1", "claims/" + session1 + "/claim.json", "claims/second/claim.json"} + answers := sameOps(t, nil, + declareOp(), + markerOp{"op": "attempt", "outcome": "accepted", "task_id": "task-a"}, markerOp{"op": "attempt", "outcome": "accepted", "task_id": "task-b"}, + markerOp{"op": "claim", "session": session1}, markerOp{"op": "claim", "session": "second"}, + markerOp{"op": "resolution", "task": "task-a", "session": session1, "facts": everything, "reason": "disagreeing"}, + markerOp{"op": "resolution", "task": "task-b", "session": session1, "facts": everything, "reason": "disagreeing"}, + markerOp{"op": "state"}, + markerOp{"op": "declare", "dispatch": "unaccepted"}, + markerOp{"op": "attempt", "dispatch": "unaccepted", "outcome": "accepted", "task_id": "task-a"}, markerOp{"op": "attempt", "dispatch": "unaccepted", "outcome": "unknown", "task_id": "task-b"}, + markerOp{"op": "claim", "dispatch": "unaccepted", "claim_dispatch": "unaccepted", "session": session1}, markerOp{"op": "claim", "dispatch": "unaccepted", "claim_dispatch": "unaccepted", "session": "second"}, + markerOp{"op": "resolution", "dispatch": "unaccepted", "task": "task-b", "session": session1, "facts": everything, "reason": "unconfirmed"}, + markerOp{"op": "state", "dispatch": "unaccepted"}, + // The control: one agreeing resolution covering everything does clear it. + markerOp{"op": "declare", "dispatch": "agreeing"}, + markerOp{"op": "attempt", "dispatch": "agreeing", "outcome": "accepted", "task_id": "task-a"}, markerOp{"op": "attempt", "dispatch": "agreeing", "outcome": "accepted", "task_id": "task-b"}, + markerOp{"op": "claim", "dispatch": "agreeing", "claim_dispatch": "agreeing", "session": session1}, markerOp{"op": "claim", "dispatch": "agreeing", "claim_dispatch": "agreeing", "session": "second"}, + markerOp{"op": "resolution", "dispatch": "agreeing", "task": "task-a", "session": session1, "facts": everything}, + markerOp{"op": "state", "dispatch": "agreeing"}, + ) + if ok(t, answers[7]) != AmbiguousIdentity || ok(t, answers[14]) != AmbiguousIdentity || ok(t, answers[21]) != CreationAccepted { + t.Fatalf("states %v %v %v", answers[7], answers[14], answers[21]) + } +} + +func TestINT19_a_malformed_fact_is_reported_by_its_field_path(t *testing.T) { + ops := []markerOp{ + declareOp(), + markerOp{"op": "write_raw", "path": "claims/" + session1 + "/claim.json", "text": `"bare"`}, markerOp{"op": "malformed"}, + markerOp{"op": "declare", "dispatch": "array"}, markerOp{"op": "publish", "dispatch": "array", "path": "attempts/0.json", "payload": map[string]any{"outcome": "accepted", "taskId": []any{"a", "b"}}}, markerOp{"op": "malformed", "dispatch": "array"}, + markerOp{"op": "declare", "dispatch": "null"}, markerOp{"op": "publish", "dispatch": "null", "path": "resolutions/0.json", "payload": map[string]any{"chosenTaskId": nil, "chosenSessionId": session1, "adjudicated": []any{}}}, markerOp{"op": "malformed", "dispatch": "null"}, + markerOp{"op": "declare", "dispatch": "nested"}, markerOp{"op": "publish", "dispatch": "nested", "path": "resolutions/0.json", "payload": map[string]any{"chosenTaskId": task1, "chosenSessionId": session1, "adjudicated": []any{"not-a-record"}}}, markerOp{"op": "malformed", "dispatch": "nested"}, + } + // Raw JSON text, so 1.0 stays a float on both sides. + for i, value := range []string{"true", "1.0", `"1"`, "null", "0", "-1", "1", ""} { + dispatch := "stamp-" + string(rune('a'+i)) + stamp := "" + if value != "" { + stamp = `, "executionGeneration": ` + value + } + ops = append(ops, markerOp{"op": "declare", "dispatch": dispatch}, markerOp{"op": "write_raw", "dispatch": dispatch, "path": "relationship.json", "text": `{"relationshipId": "` + rel1 + `"` + stamp + `, "at": "` + intentT0 + `"}`}, markerOp{"op": "malformed", "dispatch": dispatch}) + } + answers := sameOps(t, nil, ops...) + want := []any{"claims", "attempts.taskId", "resolutions.chosenTaskId", "resolutions.adjudicated"} + for k, i := range []int{2, 5, 8, 11} { + if ok(t, answers[i]) != want[k] { + t.Fatalf("answer %d %v", i, answers[i]) + } + } + for k := range 8 { + got := ok(t, answers[12+3*k+2]) + if (k < 6 && got != "relationship.executionGeneration") || (k >= 6 && got != nil) { + t.Fatalf("stamp %d %v", k, got) + } + } +} + +func TestINT20_a_counter_that_is_not_a_count_is_corruption(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "counters", "value": map[string]any{"holdsThisTurn": 0}}, + markerOp{"op": "counters", "value": nil}, + markerOp{"op": "counters", "value": map[string]any{"holdsThisTurn": nil}}, + markerOp{"op": "counters", "value": map[string]any{"holdsThisTurn": -1}}, + markerOp{"op": "counters", "value": map[string]any{"holdsThisTurn": "1"}}, + markerOp{"op": "counters", "value": map[string]any{"holdsThisTurn": true}}, + markerOp{"op": "counters", "value": []any{}}, + ) + for i, want := range []any{nil, nil, "counters.holdsThisTurn", "counters.holdsThisTurn", "counters.holdsThisTurn", "counters.holdsThisTurn", "counters"} { + if ok(t, answers[i]) != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT21_an_identity_used_as_a_directory_name_is_refused_at_every_writer(t *testing.T) { + ops := []markerOp{declareOp()} + bad := []string{"../escape", "a/b", "..", ".", "", " ", "../../escaped", "a\\b"} + for _, b := range bad { + ops = append(ops, + markerOp{"op": "claim", "session": b}, + markerOp{"op": "disposition", "session": b, "turn": "turn-1", "outcome": "interrupted"}, + markerOp{"op": "disposition", "session": session1, "turn": b, "outcome": "interrupted"}) + } + ops = append(ops, markerOp{"op": "exists_in", "path": "claims"}, markerOp{"op": "exists_in", "path": "dispositions"}, markerOp{"op": "exists", "path": "escaped"}, markerOp{"op": "exists", "path": "markers/escaped"}) + answers := sameOps(t, nil, ops...) + for i := 1; i <= 3*len(bad); i++ { + if reasonOf(t, answers[i]) != UnboundGeneration { + t.Fatalf("answer %d %v", i, answers[i]) + } + } + for _, a := range answers[len(answers)-4:] { + if ok(t, a) != false { + t.Fatalf("written: %v", answers[len(answers)-4:]) + } + } +} + +func TestINT22_a_disposition_outside_the_vocabulary_is_refused(t *testing.T) { + answers := sameOps(t, nil, declareOp(), + markerOp{"op": "disposition", "session": session1, "turn": "turn-1", "outcome": "done"}, + markerOp{"op": "disposition", "session": session1, "turn": "turn-1", "outcome": "interrupted"}, + markerOp{"op": "disposition", "session": session1, "turn": "turn-1", "outcome": "interrupted"}, + markerOp{"op": "disposition", "session": session1, "turn": "turn-1", "outcome": "failed"}, + ) + if reasonOf(t, answers[1]) != OutcomeInconsistent { + t.Fatalf("refusal %v", answers[1]) + } + for i, want := range map[int]string{2: Published, 3: Unchanged, 4: Conflict} { + if ok(t, answers[i]).(map[string]any)["published"] != want { + t.Fatalf("answer %d %v", i, answers[i]) + } + } +} + +func TestINT23_assignment_selection_for_a_workspace(t *testing.T) { + answers := sameOps(t, nil, + // No intent: not selectable. + markerOp{"op": "write_raw", "dispatch": "half-built", "path": "attempts/.keep", "text": ""}, + declareOp(), + markerOp{"op": "select", "session": session1}, + // A claim is consulted before recency. + markerOp{"op": "claim", "session": session1}, + markerOp{"op": "declare", "dispatch": "dispatch-request-2", "issue_key": "REL-2", "declared_at": intentT5}, + markerOp{"op": "select", "session": session1}, + // Without a claim, the newest declaration wins. + markerOp{"op": "select", "session": "stranger"}, + // Instants, not printed strings. + markerOp{"op": "declare", "dispatch": "offset-earlier", "declared_at": "2026-01-02T01:00:00+02:00"}, + markerOp{"op": "declare", "dispatch": "offset-later", "declared_at": "2026-01-02T00:30:00+00:00"}, + markerOp{"op": "select", "session": "stranger"}, + // An unmanaged workspace selects nothing. + markerOp{"op": "select", "session": session1, "workspace": "/elsewhere"}, + ) + name := func(i int) any { return ok(t, answers[i]).(map[string]any)["assignment"] } + if name(2) != AssignmentID("dispatch-request-1") || name(5) != AssignmentID("dispatch-request-1") || name(6) != AssignmentID("dispatch-request-2") || name(9) != AssignmentID("offset-later") || name(10) != nil { + t.Fatalf("selections %v %v %v %v %v", name(2), name(5), name(6), name(9), name(10)) + } +} + +// The publish path confines every marker write: a symlinked parent inside the subtree cannot +// carry a create-once write outside the root (marker.confined; the ValueError text is Python's). +func TestINT21_confinement_refuses_a_symlinked_parent_outside_the_root(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "write_raw", "target": "/outside/.keep", "text": ""}, + markerOp{"op": "write_raw", "target": "/markers/.keep", "text": ""}, + markerOp{"op": "symlink", "to": "/outside", "link": "/markers/escape"}, + markerOp{"op": "publish", "target": "/markers/escape/x/fact.json", "payload": map[string]any{"a": 1}, "confined": true}, + markerOp{"op": "exists", "path": "outside/x"}, + ) + if msg, _ := answers[3].(map[string]any)["error"].(string); !strings.HasPrefix(msg, "ValueError: refusing to write outside the marker root") || ok(t, answers[4]) != false { + t.Fatalf("confinement %v %v", answers[3], answers[4]) + } +} diff --git a/internal/relay/delivery/lifecycle.go b/internal/relay/delivery/lifecycle.go new file mode 100644 index 00000000..6348941c --- /dev/null +++ b/internal/relay/delivery/lifecycle.go @@ -0,0 +1,104 @@ +package delivery + +import ( + "context" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Withhold reasons (lifecycle.py). +const ( + RecipientArchived = "recipient_archived" + RecipientPaused = "recipient_paused" + RecipientUsageLimited = "recipient_usage_limited" + RecipientBudgetLimited = "recipient_budget_limited" + RecipientCannotAccept = "recipient_cannot_accept_input" + RecipientBusy = "recipient_busy" + RecipientSystemError = "recipient_system_error" + LifecycleUnknown = "lifecycle_unknown" +) + +var blockingGoalStatus = map[string]string{"paused": RecipientPaused, "usageLimited": RecipientUsageLimited, "budgetLimited": RecipientBudgetLimited} + +// Lifecycle is lifecycle.Lifecycle: what the host says about a recipient right now. +type Lifecycle struct { + TaskID string + RuntimeStatus any + Archived *bool + GoalStatus any + CanAcceptInput *bool + Deliverable string + WithholdReason any + Detail string +} + +func (l Lifecycle) MaySend() bool { return l.Deliverable == "yes" } +func (l Lifecycle) IsBusy() bool { return l.Deliverable == "busy" } + +// Observe is lifecycle.observe: read the host, decide, and never guess. +func Observe(adapter Adapter, task string, cwd any, requireEvidence bool) Lifecycle { + var runtime, goal any + var archived, accepts *bool + var problems []string + if facts, err := adapter.ReadThread(task); err != nil { + problems = append(problems, "thread read failed: "+errorLabel(err)) + } else { + runtime, accepts = facts.RuntimeStatus, facts.CanAcceptInput + } + if value, err := adapter.IsArchived(task, cwd); err != nil { + problems = append(problems, "archived check failed: "+errorLabel(err)) + } else { + archived = value + } + if value, err := adapter.ReadGoalStatus(task); err != nil { + problems = append(problems, "goal read failed: "+errorLabel(err)) + } else { + goal = value + } + detail := strings.Join(problems, "; ") + decide := func(state string, reason any) Lifecycle { + return Lifecycle{task, runtime, archived, goal, accepts, state, reason, detail} + } + goalText, _ := goal.(string) + switch { + case archived != nil && *archived: + return decide("no", RecipientArchived) + case blockingGoalStatus[goalText] != "": + return decide("no", blockingGoalStatus[goalText]) + case accepts != nil && !*accepts: + return decide("no", RecipientCannotAccept) + case runtime == "systemError": + return decide("no", RecipientSystemError) + case runtime == "active": + return decide("busy", RecipientBusy) + case len(problems) > 0 && requireEvidence: + return decide("unknown", LifecycleUnknown) + case archived == nil && requireEvidence: + return decide("unknown", LifecycleUnknown) + case runtime == "idle" || runtime == "notLoaded": + return decide("yes", nil) + case runtime == nil && !requireEvidence: + return decide("yes", nil) + } + return decide("unknown", LifecycleUnknown) +} + +func boolInt(b *bool) any { + if b == nil { + return nil + } + if *b { + return int64(1) + } + return int64(0) +} + +// RecordLifecycle is lifecycle.record. +func RecordLifecycle(ctx context.Context, s *store.Store, clock Clock, l Lifecycle) error { + return s.Transaction(ctx, func(ctx context.Context, _ *sqlConn) error { + _, err := execSQL(ctx, s, `INSERT INTO recipient_lifecycle (task_id, runtime_status, archived, goal_status, can_accept_input, deliverable, withhold_reason, detail, observed_at) VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(task_id) DO UPDATE SET runtime_status=excluded.runtime_status, archived=excluded.archived, goal_status=excluded.goal_status, can_accept_input=excluded.can_accept_input, deliverable=excluded.deliverable, withhold_reason=excluded.withhold_reason, detail=excluded.detail, observed_at=excluded.observed_at`, + l.TaskID, l.RuntimeStatus, boolInt(l.Archived), l.GoalStatus, boolInt(l.CanAcceptInput), l.Deliverable, l.WithholdReason, l.Detail, clock.ISO()) + return err + }) +} diff --git a/internal/relay/delivery/literals_test.go b/internal/relay/delivery/literals_test.go new file mode 100644 index 00000000..a55527f9 --- /dev/null +++ b/internal/relay/delivery/literals_test.go @@ -0,0 +1,121 @@ +package delivery + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/faults" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/mergeturn" +) + +// Every reason, hold, state and next-action word this package emits that is NOT a member of the +// frozen errors.RefusalReason enum (contract/schema/relay-exit-codes.json). Each stays exact: +// the test asserts the Python source spells it as a quoted literal in the named module. +var literalWords = map[string][]string{ + "cli.py": {"state_directory_serves_another_socket", "ambiguous_state_directory", "unidentified_state_directory"}, + "ack.py": {"delivery_unconfirmed", "delivery_state_changed", "ack_predates_attempt", "already_settled", "replaced", "changed", "withheld", "verified", "unverified_turn", "host_read", "unrecorded", "revision_mismatch", "proceed", "already_claimed"}, + "policy.py": {"attempt_cap", "busy_cap", "push_channel_closed", "host_lost_turn", "turn_check_undecided", "unknown_send_lost", "unknown_send_undecided", "unknown_send_hold_named", "min_send_interval", "hourly_cap"}, + "lifecycle.py": {"recipient_archived", "recipient_paused", "recipient_usage_limited", "recipient_budget_limited", "recipient_cannot_accept_input", "recipient_busy", "recipient_system_error", "lifecycle_unknown"}, + "transport.py": {"accepted", "failed", "outcome_unknown", "in_progress_or_unknown", "dispatched", "deferred_busy", "withheld_pre_send", "held_uncertain", "inbox_only", "queued", "acknowledged", "superseded", "sending", "settings_not_preserved", "setting_unobservable", "environments_unknown", "unverifiable_permission_profile", "settings_differ_after_load"}, + "currency.py": {"sole_revision", "declared_chain", "no_revision", "fork", "cycle", "unknown_predecessor", "disconnected"}, + "delivery.py": {"completion_event", "revision_request", "merge_turn_grant", "delivery_presend_withheld", "delivery_settings_noted", "dispatched_awaiting_ack", "stored_not_woken", "held_uncertain_awaiting_evidence", "awaiting_ack", "awaiting_child_receipt", "parent_busy", "channel_closed", "turn_accepted", "settings_rejected", "in_flight", "awaiting_send", "awaiting_receipt", "transport_accepted", "settings_check", "lifecycle_read", "parent_busy", "delivery_withheld_inactive", "delivery_withheld", "delivery_deferred_busy", "delivery_attempted", "delivery_attempt_settled_elsewhere", "delivery_queued", "delivery_superseded", "delivery_recipient_resolved", "restoration_attempted", "relationship_row", "linkage"}, + "reconcile.py": {"turn_found", "receipt_turn_id", "confirmed_pre_send_rejection", "none", "reconciled"}, + "assignment.py": {"daemon_reconciles_delivery", "daemon_confirms_correction", "parent_recovers_held_correction", "parent_reads_child_disposition", "parent_recovers_unknown_send_lost", "parent_recovers_unknown_send_undecided"}, + "hostloss.py": {"present", "unknown", "listing_bounded", "listing_empty", "token_scan_bounded", "token_without_turn", "token_in_other_item", "no_send_time", "no_turn", "receipt_unsettled", "receipt_missing", "settled", "unsettled", "missing", "not_moved", "report_only", "held"}, + "criteria.py": {"managed", "legacy", "covered", "legacy_unregistered"}, + "restoration.py": {"carried", "truncated", "not_carried", "unmeasured", "relay-message/legacy"}, + "settings.py": {"approval_policy_differs_from_record", "runtime_roots_narrower_than_record"}, + "registry.py": {"anchor_pending", "bound", "needs_changes_revision", "initial_assignment", "anchor_conflict", "anchor_bound", "generation_opened", "status_changed"}, + "supervisorchannel.py": {"turn_predates_send"}, + "sync.py": {"coordination_document", "sync_enqueued", "pending", "claimed", "confirmed"}, + "intent.py": {"relationship_registered", "identity_bound", "ambiguous_identity", "intent_expired", "creation_unknown", "creation_accepted", "intent_declared", "bound", "unchanged", "conflict", "current", "stale", "absent", "accepted", "unknown", "failed", "in_progress", "blocked_needs_input", "interrupted", "ready_for_review"}, + "marker.py": {"published", "exists", "flag", "env", "xdg", "home"}, + "declarations.py": {"declarations/1", "recorded", "unchanged", "conflict", "not_recorded", "failed", "store_disagrees", "no_store_recorded", "store_absent", "store_unreadable", "store_not_a_file", "store_unopenable", "store_write_failed", "store_locked"}, + "hostloss.py#b2": {"queued"}, + "daemon.py": {"turnsLost", "turnsUndecided"}, + "assignment.py#b2": {"parent_acknowledges", "daemon_verifies_acknowledgement", "parent_reacknowledges", "daemon_redelivers_host_lost_turn", "parent_recovers_host_lost_turn", "operator_changes_send_policy", "daemon_delivers_correction", "parent_verifies", "daemon_delivers"}, + "delivery.py#b2": {"dispatched_awaiting_grant_acknowledgement", "grant_acknowledged", "prepared", "confirmed_unsent", "uncertain", "unavailable", "redelivering:", "awaiting_ack:", "awaiting_send:"}, + "reconcile.py#b2": {"grant_acknowledged", "recipientTrace", "recipientTurn", "undecidedChanged"}, + "assignment.py#b3": {"turnCheck", "hostLostAttempts"}, + "mergeturn.py": {"merge_turn_absent", "merge_turn_closed", "merge_turn_regranted", "merge_turn_grant_answered", "merge_turn_grant_unreadable", "grant_acknowledged", "holding", "merging"}, + "faultsweep.py": {"delivery_stalled", "delivery_retrying", "recovered"}, + "faults.py": {"open_record", "append_comment", "blocking", "observed", "open", "broken", "degraded", "occurrence", "escalate", "team+project", "none"}, + "bridge_adapter.py": {"thread/turns/list", "thread/items/list", "notLoaded", "desc", "asc", "nextCursor", "startedAt"}, + "cli.py#marker": {"marker_unreadable", "marker_malformed", "claim_not_standing", "claim_uncorrelated", "store_changed"}, +} + +func TestLiteralReasons_outside_the_frozen_enum_are_spelled_as_python_spells_them(t *testing.T) { + enum := map[string]bool{} + schema, err := os.ReadFile(filepath.Join(repoRoot(t), "contract", "schema", "relay-exit-codes.json")) + mustDo(t, err) + for _, r := range refusalEnum(t, schema) { + enum[r] = true + } + src := filepath.Join(repoRoot(t), "packages", "codex-session-relay", "src", "codex_session_relay") + checked := 0 + for file, words := range literalWords { + raw, err := os.ReadFile(filepath.Join(src, strings.Split(file, "#")[0])) + mustDo(t, err) + text := string(raw) + for _, w := range words { + if !strings.Contains(text, `"`+w+`"`) && !strings.Contains(text, `'`+w+`'`) { + t.Errorf("%s does not spell %q as a literal", file, w) + } + checked++ + } + } + // And the Go constants carrying them are the same words. + for _, pair := range [][2]string{ + {AttemptCap, "attempt_cap"}, {PushChannelClosed, "push_channel_closed"}, {HostLostTurn, "host_lost_turn"}, {HourlyCap, "hourly_cap"}, + {LifecycleUnknown, "lifecycle_unknown"}, {RecipientArchived, "recipient_archived"}, {DeliveryUnconfirmed, "delivery_unconfirmed"}, + {AckPredatesAttempt, "ack_predates_attempt"}, {PresendWithheld, "delivery_presend_withheld"}, {SettingsNoted, "delivery_settings_noted"}, + {Sole, "sole_revision"}, {Chain, "declared_chain"}, {TurnFound, "turn_found"}, {ApprovalDiffersFromRecord, "approval_policy_differs_from_record"}, + {TurnPredatesSend, "turn_predates_send"}, {SettingsDifferAfterLoad, "settings_differ_after_load"}, + {RelationshipRegistered, "relationship_registered"}, {IdentityBound, "identity_bound"}, {AmbiguousIdentity, "ambiguous_identity"}, {IntentExpired, "intent_expired"}, + {CreationUnknown, "creation_unknown"}, {CreationAccepted, "creation_accepted"}, {IntentDeclared, "intent_declared"}, {Bound, "bound"}, {Unchanged, "unchanged"}, {Conflict, "conflict"}, + {DispatchCurrent, "current"}, {DispatchStale, "stale"}, {DispatchAbsent, "absent"}, {declarationsCapability, "declarations/1"}, {declRecorded, "recorded"}, + {declNotRecordedState, "not_recorded"}, {declFailed, "failed"}, {Published, "published"}, {Exists, "exists"}, + {TurnCheckUndecided, "turn_check_undecided"}, {UnknownSendLost, "unknown_send_lost"}, {UnknownSendUndecided, "unknown_send_undecided"}, {UnknownSendHoldNamed, "unknown_send_hold_named"}, + {Requeued, "queued"}, {HeldRedelivery, "held"}, {NotMoved, "not_moved"}, {ReportOnly, "report_only"}, {ListingBoundedW, "listing_bounded"}, {ListingEmptyW, "listing_empty"}, + {TokenScanBounded, "token_scan_bounded"}, {TokenWithoutTurn, "token_without_turn"}, {TokenInOtherItem, "token_in_other_item"}, {NoSendTime, "no_send_time"}, {NoTurn, "no_turn"}, + {ReceiptUnsettled, "receipt_unsettled"}, {ReceiptMissing, "receipt_missing"}, {reconcileAction, "daemon_reconciles_delivery"}, {unknownSendHeldAction, "parent_recovers_unknown_send_lost"}, + {unknownSendUndecidedAction, "parent_recovers_unknown_send_undecided"}, {correctionHeld, "parent_recovers_held_correction"}, {correctionAnswered, "parent_reads_child_disposition"}, + {mergeturn.Absent, "merge_turn_absent"}, {mergeturn.Closed, "merge_turn_closed"}, {mergeturn.Regranted, "merge_turn_regranted"}, {mergeturn.GrantAnswered, "merge_turn_grant_answered"}, {mergeturn.GrantUnreadable, "merge_turn_grant_unreadable"}, + {faults.Broken, "broken"}, {faults.Degraded, "degraded"}, {faults.Open, "open"}, {faults.Observed, "observed"}, + } { + if pair[0] != pair[1] { + t.Errorf("constant %q != %q", pair[0], pair[1]) + } + } + // Every refusal reason the package raises IS an enum member (or one of the cli.py literals). + for _, r := range []string{NotClaimable, RecipientNotAuthorized, ScopeEscape, RelationshipNotActive, UnregisteredRelationship, UnknownGeneration, RelationUnreadable, RelationOwnerDrift, DuplicateScopeOwner, LinkConflict, UnregisteredScope, + SettingsUnavailable, SettingsIncomplete, SettingsMistyped, UnsupportedSandboxType, UnsupportedApprovalPolicy, RolePolicyUnconfigured, RoleBindingMismatch, CriteriaUnregistered, CriteriaNotCovered, CriteriaSetChanged, + UnknownCriterion, FindingsRequired, DispositionConflict, ReviewNotBound, WrongDeliveryKind, AckProofMismatch, AckTurnUnverified, NotAcknowledged, RestorationUndeliverable, StaleGeneration, SupersededRevision, RevisionAmbiguous, SyncNotClaimable, + RelationshipConflict, UnboundGeneration, OutcomeInconsistent} { + if !enum[r] { + t.Errorf("%q is raised as a refusal but is not in the frozen enum", r) + } + } + if checked < 145 { + t.Fatalf("only %d literals checked", checked) + } +} + +// refusalEnum reads the frozen RefusalReason values (refusalReasons: NAME -> value). +func refusalEnum(t *testing.T, raw []byte) []string { + var doc struct { + RefusalReasons map[string]string `json:"refusalReasons"` + } + mustDo(t, json.Unmarshal(raw, &doc)) + var out []string + for _, v := range doc.RefusalReasons { + out = append(out, v) + } + if len(out) < 100 { + t.Fatalf("read %d enum members", len(out)) + } + return out +} diff --git a/internal/relay/delivery/main_test.go b/internal/relay/delivery/main_test.go new file mode 100644 index 00000000..38a7e4a8 --- /dev/null +++ b/internal/relay/delivery/main_test.go @@ -0,0 +1,24 @@ +package delivery + +import ( + "os" + "path/filepath" + "testing" +) + +// TestMain removes the crw binary the CLI tests build and the Python capture trees, once per +// package run. +func TestMain(m *testing.M) { + code := m.Run() + for _, path := range captureCleanups { + if err := os.RemoveAll(path); err != nil { + code = 1 + } + } + if crwPath != "" { + if err := os.RemoveAll(filepath.Dir(crwPath)); err != nil { + code = 1 + } + } + os.Exit(code) +} diff --git a/internal/relay/delivery/marker.go b/internal/relay/delivery/marker.go new file mode 100644 index 00000000..e4a8d84f --- /dev/null +++ b/internal/relay/delivery/marker.go @@ -0,0 +1,503 @@ +package delivery + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "syscall" + "unicode/utf8" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The managed marker (marker.py): create-once facts on a filesystem the relay database never +// sees. Every fact is published once, by writing a sibling temp, fsyncing it and linking it onto +// the target, so first-publication-wins is an operating-system fact; the layout and every rule +// here are fixed by skills/crw-run/references/hook-contract.md. + +// Marker words and names (marker.py). +const ( + MarkerEnv = "CODEX_SESSION_RELAY_MARKER_ROOT" + markerDirectoryName = "codex-session-marker" + Published = "published" + Exists = "exists" + claimFile = "claim.json" +) + +// MarkerPrecedence is PRECEDENCE: the order the marker root rules are asked in. +var MarkerPrecedence = []string{"flag", "env", "xdg", "home"} + +var singleFacts = []struct{ key, name string }{{"intent", "intent.json"}, {"bound", "bound.json"}, {"relationship", "relationship.json"}} +var numberedFacts = []string{"attempts", "conflicts", "resolutions"} + +var assignmentPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// ValidAssignment is valid_assignment: the hex sha256 of a dispatch request id. +func ValidAssignment(assignment any) bool { + text, _ := assignment.(string) + return assignmentPattern.MatchString(text) +} + +// Named is named: only a string with a non-blank character (str.strip) names an identity. +func Named(value any) bool { + text, ok := value.(string) + return ok && strings.TrimFunc(text, isPySpace) != "" +} + +// isPySpace is str.isspace for one character, the set str.strip removes. +func isPySpace(r rune) bool { + switch r { + case ' ', '\t', '\n', '\v', '\f', '\r', 0x1c, 0x1d, 0x1e, 0x1f, 0x85, 0xa0, 0x1680, 0x2028, 0x2029, 0x202f, 0x205f, 0x3000: + return true + } + return r >= 0x2000 && r <= 0x200a +} + +// SameIdentity is same_identity: unnamed on either side is never a match. +func SameIdentity(left, right any) bool { + return Named(left) && Named(right) && left == right +} + +// ValidSegment is valid_segment: an identity that may be used as a directory name. +func ValidSegment(value any) bool { + if !Named(value) { + return false + } + text := value.(string) + if text == "." || text == ".." { + return false + } + return !strings.ContainsAny(text, "/\\\x00") +} + +// MarkerSelection is marker.MarkerSelection: which rule chose the root, and the value that won. +type MarkerSelection struct{ Path, Source, Detail string } + +// Record is MarkerSelection.to_record. +func (m MarkerSelection) Record() Obj { + precedence := make([]any, len(MarkerPrecedence)) + for i, p := range MarkerPrecedence { + precedence[i] = p + } + return Obj{{Key: "path", Value: m.Path}, {Key: "source", Value: m.Source}, {Key: "detail", Value: m.Detail}, {Key: "precedence", Value: precedence}} +} + +// ResolveMarkerRoot is resolve_marker_root: flag, then the environment, then XDG, then home. +// Deliberately a different directory from the relay state directory. +func ResolveMarkerRoot(explicit string) (MarkerSelection, error) { + if explicit != "" { + path, err := absoluteUser(explicit) + return MarkerSelection{path, "flag", "--marker-root " + explicit}, err + } + if override := os.Getenv(MarkerEnv); override != "" { + path, err := absoluteUser(override) + return MarkerSelection{path, "env", MarkerEnv + "=" + override}, err + } + var base, source, detail string + if xdg := os.Getenv("XDG_STATE_HOME"); xdg != "" { + expanded, err := store.ExpandUser(xdg) + if err != nil { + return MarkerSelection{}, err + } + base, source, detail = expanded, "xdg", "XDG_STATE_HOME="+xdg + } else { + home, err := os.UserHomeDir() + if err != nil { + return MarkerSelection{}, err + } + base, source = filepath.Join(home, ".local", "state"), "home" + detail = "default under " + base + } + path, err := absolutePath(base + "/" + markerDirectoryName) + return MarkerSelection{path, source, detail}, err +} + +// absoluteUser is Path(value).expanduser().absolute(): no symlink is resolved and no ".." folded. +func absoluteUser(value string) (string, error) { + expanded, err := store.ExpandUser(value) + if err != nil { + return "", err + } + return absolutePath(expanded) +} + +// absolutePath is Path.absolute(): the pathlib spelling, prefixed by the cwd when relative. +func absolutePath(value string) (string, error) { + if !strings.HasPrefix(value, "/") { + cwd, err := os.Getwd() + if err != nil { + return "", err + } + value = cwd + "/" + value + } + parts := []string{} + for _, part := range strings.Split(value, "/") { + if part != "" && part != "." { + parts = append(parts, part) + } + } + return "/" + strings.Join(parts, "/"), nil +} + +// resolved is Path(value).expanduser().resolve(): symlinks followed, a missing tail kept. +func resolved(value string) (string, error) { + expanded, err := store.ExpandUser(value) + if err != nil { + return "", err + } + return store.ResolvePath(expanded) +} + +func sha256Hex(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:]) +} + +// WorkspaceKey is workspace_key: sha256 of the resolved workspace path, so a symlinked or +// relative cwd reaches the same assignment the coordinator declared against. +func WorkspaceKey(workspace string) (string, error) { + path, err := resolved(workspace) + if err != nil { + return "", err + } + return sha256Hex(path), nil +} + +// AssignmentID is assignment_id: the hash of the dispatch request id, never the id itself. +func AssignmentID(dispatchRequestID string) string { return sha256Hex(dispatchRequestID) } + +// WorkspaceDir is workspace_dir. +func WorkspaceDir(root, workspace string) (string, error) { + key, err := WorkspaceKey(workspace) + if err != nil { + return "", err + } + return filepath.Join(root, key), nil +} + +// AssignmentDir is assignment_dir: the assignment owns a level under its workspace. A value that +// is not an assignment id is ValueError, as _checked_assignment raises it. +func AssignmentDir(root, workspace string, assignment any) (string, error) { + if !ValidAssignment(assignment) { + return "", &hostError{"ValueError", "an assignment id is the hex sha256 of a dispatch request id, not " + pyReprValue(assignment)} + } + directory, err := WorkspaceDir(root, workspace) + if err != nil { + return "", err + } + return filepath.Join(directory, assignment.(string)), nil +} + +// FactDigest is fact_digest: SHA-256 over the fact's canonical JSON with factId removed. +func FactDigest(payload Obj) string { + body := Obj{} + for _, f := range payload { + if f.Key != "factId" { + body = append(body, f) + } + } + return sha256Hex(canonical(body)) +} + +// canonical is _canonical: json.dumps(sort_keys=True, separators=(",", ":")), ASCII escaped. +func canonical(value any) string { + var b strings.Builder + writeCanonical(&b, value) + return b.String() +} + +func writeCanonical(b *strings.Builder, value any) { + switch v := value.(type) { + case Obj: + fields := append(Obj(nil), v...) + sort.SliceStable(fields, func(i, j int) bool { return fields[i].Key < fields[j].Key }) + b.WriteByte('{') + for i, f := range fields { + if i > 0 { + b.WriteByte(',') + } + writeString(b, f.Key) + b.WriteByte(':') + writeCanonical(b, f.Value) + } + b.WriteByte('}') + case []any: + b.WriteByte('[') + for i, item := range v { + if i > 0 { + b.WriteByte(',') + } + writeCanonical(b, item) + } + b.WriteByte(']') + default: + writeJSON(b, v, true) + } +} + +// fsyncDirectory is _fsync_directory: best effort, the link already decided who won. +func fsyncDirectory(directory string) { + handle, err := os.Open(directory) + if err != nil { + return + } + _ = handle.Sync() + _ = handle.Close() +} + +// Publish is marker.publish: create-once publication, "published" when this writer won and +// "exists" when it lost. root, when not empty, confines the write under it before and after the +// directory is made. +func Publish(target string, payload any, root string) (string, error) { + directory := filepath.Dir(target) + if root != "" { + if _, err := confined(directory, root); err != nil { + return "", err + } + } + if err := os.MkdirAll(directory, 0o700); err != nil { + return "", err + } + if root != "" { + if _, err := confined(directory, root); err != nil { + return "", err + } + } + suffix := make([]byte, 6) + if _, err := rand.Read(suffix); err != nil { + return "", err + } + temp := filepath.Join(directory, "."+filepath.Base(target)+".tmp."+strconv.Itoa(os.Getpid())+"."+hex.EncodeToString(suffix)) + body := []byte(canonical(payload)) + handle, err := os.OpenFile(temp, os.O_CREATE|os.O_EXCL|os.O_WRONLY|syscall.O_NOFOLLOW, 0o600) + if err != nil { + return "", err + } + if _, err := handle.Write(body); err != nil { + _ = handle.Close() + _ = os.Remove(temp) + return "", err + } + if err := handle.Sync(); err != nil { + _ = handle.Close() + _ = os.Remove(temp) + return "", err + } + if err := handle.Close(); err != nil { + _ = os.Remove(temp) + return "", err + } + outcome := Published + linkErr := os.Link(temp, target) + _ = os.Remove(temp) + switch { + case errors.Is(linkErr, fs.ErrExist): + outcome = Exists + case linkErr != nil: + return "", linkErr + } + fsyncDirectory(directory) + return outcome, nil +} + +// Fact read answers (_read_fact): present, absent, or unreadable. +const ( + factPresent = iota + factAbsent + factUnreadable +) + +// readFact is _read_fact: "it is not there" and "I could not look" are different answers. +func readFact(path string) (any, int) { + data, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + return nil, factAbsent + } + if err != nil || !utf8.Valid(data) { + return nil, factUnreadable + } + value, err := loads(string(data)) + if err != nil { + return nil, factUnreadable + } + return value, factPresent +} + +// listing is marker.listing: every entry of a directory, sorted, and whether it could be read. +// FileNotFound is the only absence. only is "" (any), "directories" or "entries". +func listing(directory, pattern, only string) ([]string, bool) { + entries, err := os.ReadDir(directory) + if errors.Is(err, fs.ErrNotExist) { + return nil, true + } + if err != nil { + return nil, false + } + var found []string + for _, entry := range entries { + if pattern != "" { + if matched, _ := filepath.Match(pattern, entry.Name()); !matched { + continue + } + } + path := filepath.Join(directory, entry.Name()) + if only != "" { + info, err := os.Stat(path) + isDirectory := false + if err == nil { + isDirectory = info.IsDir() + } else if !errors.Is(err, fs.ErrNotExist) { + return nil, false + } + if (only == "directories") != isDirectory { + continue + } + } + found = append(found, path) + } + sort.Strings(found) + return found, true +} + +// confined is marker.confined: resolve a path and require it to stay under the root owning it. +func confined(path, root string) (string, error) { + resolvedPath, err := store.ResolvePath(path) + if err != nil { + return "", err + } + base, err := store.ResolvePath(root) + if err != nil { + return "", err + } + if resolvedPath != base && !strings.HasPrefix(resolvedPath, strings.TrimSuffix(base, "/")+"/") { + return "", &hostError{"ValueError", "refusing to write outside the marker root: " + resolvedPath + " is not under " + base} + } + return resolvedPath, nil +} + +// identified is _identified: the factId the READER walked to, never one copied out of a body. +// A fact that is not a record reaches the reader as it is. +func identified(value any, factID string) any { + record, ok := value.(Obj) + if !ok { + return value + } + out := append(Obj(nil), record...) + return set(out, "factId", factID) +} + +func stem(name string) string { return strings.TrimSuffix(name, filepath.Ext(name)) } + +// ReadAssignment is read_assignment: every published fact, plus the labels of anything that could +// not be read. Nothing here fails; every outside-world step answers with a label instead. +func ReadAssignment(directory string) (Obj, []string) { + marker, unreadable := Obj{}, []string{} + for _, fact := range singleFacts { + value, status := readFact(filepath.Join(directory, fact.name)) + switch status { + case factAbsent: + continue + case factUnreadable: + unreadable = append(unreadable, fact.key) + continue + } + marker = append(marker, F{Key: fact.key, Value: identified(value, fact.key)}) + } + for _, key := range numberedFacts { + entries, readable := listing(filepath.Join(directory, key), "*.json", "") + if !readable { + unreadable = append(unreadable, key) + continue + } + items := []any{} + for _, entry := range entries { + name := filepath.Base(entry) + if strings.HasPrefix(name, ".") { + continue + } + value, status := readFact(entry) + switch status { + case factAbsent: + continue + case factUnreadable: + unreadable = append(unreadable, key+"/"+stem(name)) + continue + } + items = append(items, identified(value, key+"/"+stem(name))) + } + marker = append(marker, F{Key: key, Value: items}) + } + sessions, readable := listing(filepath.Join(directory, "claims"), "", "directories") + if !readable { + return marker, append(unreadable, "claims") + } + claims := []any{} + for _, session := range sessions { + factID := "claims/" + filepath.Base(session) + "/" + claimFile + value, status := readFact(filepath.Join(session, claimFile)) + switch status { + case factAbsent: + continue + case factUnreadable: + unreadable = append(unreadable, factID) + continue + } + claims = append(claims, identified(value, factID)) + } + return append(marker, F{Key: "claims", Value: claims}), unreadable +} + +// ReadDisposition is read_disposition: the disposition this session recorded for this turn, read +// at the path the Stop identity derives. (nil, true) when absent or when the identity names no +// path; (nil, false) when it could not be read. +func ReadDisposition(directory string, sessionID, turnID any) (any, bool) { + if !ValidSegment(sessionID) || !ValidSegment(turnID) { + return nil, true + } + session, turn := sessionID.(string), turnID.(string) + value, status := readFact(filepath.Join(directory, "dispositions", session, turn+".json")) + switch status { + case factAbsent: + return nil, true + case factUnreadable: + return nil, false + } + return identified(value, "dispositions/"+session+"/"+turn), true +} + +// ListAssignments is list_assignments: every assignment declared for this workspace, oldest name +// first, and whether the workspace directory could be read. +func ListAssignments(root, workspace string) ([]string, bool, error) { + directory, err := WorkspaceDir(root, workspace) + if err != nil { + return nil, false, err + } + found, readable := listing(directory, "", "directories") + return found, readable, nil +} + +// markerFactList is a numbered or claims fact list as the reader returned it. +func markerFactList(marker Obj, key string) []any { + v, _ := get(marker, key) + list, _ := v.([]any) + return list +} + +func markerFact(marker Obj, key string) Obj { + v, _ := get(marker, key) + record, _ := v.(Obj) + return record +} + +func fieldOf(record Obj, key string) any { + v, _ := get(record, key) + return v +} diff --git a/internal/relay/delivery/marker_ops_test.go b/internal/relay/delivery/marker_ops_test.go new file mode 100644 index 00000000..84fd8974 --- /dev/null +++ b/internal/relay/delivery/marker_ops_test.go @@ -0,0 +1,442 @@ +package delivery + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// markerOps runs one list of marker/intent operations through the real Python modules +// (testdata/markerops.py) and through this package over the SAME tree path, one after the +// other, and returns both answer lists. Sharing the path keeps workspace keys, assignment +// directories and every printed path identical, so the lists compare whole. +type markerOp = map[string]any + +func runMarkerOps(t *testing.T, env map[string]any, ops []markerOp) (python, golang []any) { + t.Helper() + tree := t.TempDir() + spec, err := json.Marshal(map[string]any{"ops": ops, "env": env}) + mustDo(t, err) + root := repoRoot(t) + script, _ := filepath.Abs("testdata/markerops.py") + cmd := exec.Command("uv", "run", "--no-sync", "python", script, tree) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + home := t.TempDir() + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "xs"), "XDG_DATA_HOME="+filepath.Join(home, "data"), "XDG_CONFIG_HOME="+filepath.Join(home, "config"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, "CODEX_SESSION_RELAY_MARKER_ROOT=") + cmd.Stdin = strings.NewReader(string(spec)) + output, err := cmd.Output() + if err != nil { + stderr := "" + var exit *exec.ExitError + if errors.As(err, &exit) { + stderr = string(exit.Stderr) + } + t.Fatalf("python marker ops: %v\n%s", err, stderr) + } + mustDo(t, json.Unmarshal(output, &python)) + entries, err := os.ReadDir(tree) + mustDo(t, err) + for _, entry := range entries { + mustDo(t, os.RemoveAll(filepath.Join(tree, entry.Name()))) + } + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "xs")) + t.Setenv(MarkerEnv, "") + for name, value := range env { + setEnv(t, name, value, tree) + } + d := &markerDriver{t: t, tree: tree, ctx: context.Background()} + defer d.close() + mustDo(t, os.MkdirAll(d.work(), 0o755)) + for _, op := range ops { + golang = append(golang, d.answer(op)) + } + return python, golang +} + +func setEnv(t *testing.T, name string, value any, tree string) { + t.Setenv(name, "") + if value == nil { + mustDo(t, os.Unsetenv(name)) + return + } + mustDo(t, os.Setenv(name, strings.ReplaceAll(value.(string), "", tree))) +} + +type markerDriver struct { + t *testing.T + tree string + ctx context.Context + s *store.Store + held *sql.Conn +} + +func (d *markerDriver) root() string { return filepath.Join(d.tree, "markers") } +func (d *markerDriver) work() string { return filepath.Join(d.tree, "work") } +func (d *markerDriver) path(v any) string { + text, _ := v.(string) + return strings.ReplaceAll(text, "", d.tree) +} + +func (d *markerDriver) close() { + if d.held != nil { + _ = d.held.Close() + } + if d.s != nil { + _ = d.s.Close() + } +} + +func (d *markerDriver) store() *store.Store { + if d.s == nil { + s, err := store.Open(d.ctx, filepath.Join(d.tree, "state", "relay.sqlite3"), "") + mustDo(d.t, err) + d.s = s + } + return d.s +} + +func opString(op markerOp, key, fallback string) string { + if v, ok := op[key].(string); ok { + return v + } + return fallback +} + +func (d *markerDriver) assignment(op markerOp) any { + if v, ok := op["assignment"]; ok { + return v + } + return AssignmentID(opString(op, "dispatch", "dispatch-request-1")) +} + +func (d *markerDriver) adir(op markerOp) string { + dir, err := AssignmentDir(d.root(), d.work(), AssignmentID(opString(op, "dispatch", "dispatch-request-1"))) + mustDo(d.t, err) + return dir +} + +// fromJSON turns a decoded JSON value into this package's Obj/[]any/int64 shapes. +func fromJSON(v any) any { + raw, _ := json.Marshal(v) + out, _ := loads(string(raw)) + return out +} + +func (d *markerDriver) answer(op markerOp) any { + value, err := d.run(op) + var refused *store.RefusedError + var host *hostError + switch { + case errors.As(err, &refused): + return map[string]any{"reason": refused.Reason, "detail": refused.Detail} + case errors.As(err, &host): + return map[string]any{"error": host.Error()} + case err != nil: + d.t.Fatalf("op %v: %v", op, err) + } + return map[string]any{"ok": value} +} + +func (d *markerDriver) run(op markerOp) (any, error) { + const t0 = "2026-01-01T00:00:00+00:00" + at := opString(op, "at", t0) + switch op["op"] { + case "declare": + decl := IntentDeclaration{Workspace: d.work(), DispatchRequestID: opString(op, "dispatch", "dispatch-request-1"), IssueKey: opString(op, "issue_key", "REL-1"), DeclaredAt: opString(op, "declared_at", t0), + CriteriaSource: op["criteria_source"], BaselineRevision: op["baseline_revision"], AuthorizedSettings: fromJSON(op["authorized_settings"])} + if w, ok := op["workspace"]; ok { + decl.Workspace = d.path(w) + } + if p, ok := op["db_path"]; ok { + decl.DBPath = d.path(p) + } + return DeclareIntent(d.root(), decl) + case "attempt": + return RecordAttempt(d.root(), d.work(), d.assignment(op), op["outcome"].(string), at, op["task_id"]) + case "claim": + return PublishClaim(d.root(), d.work(), d.assignment(op), op["session"], opString(op, "claim_dispatch", "dispatch-request-1"), "turn-1", at) + case "bind": + return BindIdentity(d.root(), d.work(), d.assignment(op), op["session"], op["task"], at) + case "open_generation": + rid := op["relationship_id"].(string) + generation := int64(1) + if g, ok := op["generation"]; ok { + generation = fromJSON(g).(int64) + } + current := generation + c, hasCurrent := op["current"] + if hasCurrent { + current = fromJSON(c).(int64) + } + s := d.store() + _, err := execSQL(d.ctx, s, "INSERT OR IGNORE INTO relationships (relationship_id, issue_key, status, parent_task_id, parent_host_id, child_task_id, child_host_id, execution_generation, artifact_roots, allowed_recipients, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + rid, "REL-1", "active", "01parent-task", "host-a", "01child-task", "host-a", current, "[]", "[]", t0, t0) + mustDo(d.t, err) + if hasCurrent { + _, err = execSQL(d.ctx, s, "UPDATE relationships SET execution_generation = ? WHERE relationship_id = ?", current, rid) + mustDo(d.t, err) + } + _, err = execSQL(d.ctx, s, "INSERT OR IGNORE INTO generations (relationship_id, execution_generation, dispatch_request_id, anchor_state, opened_at) VALUES (?,?,?,?,?)", rid, generation, opString(op, "generation_dispatch", "dispatch-request-1"), "bound", t0) + mustDo(d.t, err) + return nil, nil + case "register": + return RegisterRelationship(d.ctx, d.root(), d.work(), d.assignment(op), op["relationship_id"].(string), opString(op, "register_dispatch", "dispatch-request-1"), t0, d.path(opString(op, "db_path", "/state/relay.sqlite3"))) + case "hold_begin": + conn, err := d.store().DB.Conn(d.ctx) + mustDo(d.t, err) + _, err = conn.ExecContext(d.ctx, "BEGIN IMMEDIATE") + mustDo(d.t, err) + d.held = conn + return nil, nil + case "hold_end": + _, err := d.held.ExecContext(d.ctx, "ROLLBACK") + mustDo(d.t, err) + mustDo(d.t, d.held.Close()) + d.held = nil + return nil, nil + case "relay_tables": + s := d.store() + relationships, err := all(d.ctx, s, "SELECT * FROM relationships") + mustDo(d.t, err) + generations, err := all(d.ctx, s, "SELECT * FROM generations") + mustDo(d.t, err) + journal, err := one(d.ctx, s, "SELECT COUNT(*) AS n FROM journal") + mustDo(d.t, err) + return map[string]any{"relationships": relationships, "generations": generations, "journal": journal.I("n")}, nil + case "resolution": + found, _ := ReadAssignment(d.adir(op)) + var entries []Obj + pool := append(append(markerFactList(found, "attempts"), markerFactList(found, "claims")...), markerFactList(found, "conflicts")...) + ids, _ := op["facts"].([]any) + for _, id := range ids { + for _, raw := range pool { + fact := raw.(Obj) + if fieldOf(fact, "factId") == id { + digest := FactDigest(fact) + if op["digest"] == "zero" { + digest = strings.Repeat("0", 64) + } + entries = append(entries, Obj{{Key: "factId", Value: id}, {Key: "digest", Value: digest}}) + } + } + } + return PublishResolution(d.root(), d.work(), d.assignment(op), op["task"], op["session"], opOr(op, "reason", "r"), "2026-01-01T00:05:00+00:00", entries) + case "publish": + root := "" + if op["confined"] == true { + root = d.root() + } + return Publish(d.target(op), fromJSON(op["payload"]), root) + case "write_raw": + target := d.target(op) + mustDo(d.t, os.MkdirAll(filepath.Dir(target), 0o700)) + mustDo(d.t, os.WriteFile(target, []byte(op["text"].(string)), 0o600)) + return nil, nil + case "unlink": + mustDo(d.t, os.Remove(filepath.Join(d.adir(op), op["path"].(string)))) + return nil, nil + case "exists": + _, err := os.Stat(filepath.Join(d.tree, op["path"].(string))) + return err == nil, nil + case "exists_in": + _, err := os.Stat(filepath.Join(d.adir(op), op["path"].(string))) + return err == nil, nil + case "listdir": + entries, err := os.ReadDir(d.path(op["target"])) + mustDo(d.t, err) + names := []any{} + for _, e := range entries { + names = append(names, e.Name()) + } + return names, nil + case "read_file": + data, err := os.ReadFile(d.path(op["target"])) + mustDo(d.t, err) + return loads(string(data)) + case "facts": + found, unreadable := ReadAssignment(d.adir(op)) + return map[string]any{"facts": found, "unreadable": unreadable}, nil + case "state": + found, _ := ReadAssignment(d.adir(op)) + return DeriveAssignmentState(found, opOr(op, "now", "2026-01-01T00:05:00+00:00")), nil + case "malformed": + found, _ := ReadAssignment(d.adir(op)) + return noneIfEmpty(Malformed(found)), nil + case "counters": + return noneIfEmpty(MalformedCounters(fromJSON(op["value"]))), nil + case "contested": + found, _ := ReadAssignment(d.adir(op)) + return IdentityContested(found), nil + case "covered": + found, _ := ReadAssignment(d.adir(op)) + for _, raw := range markerFactList(found, "claims") { + if fieldOf(raw.(Obj), "factId") == op["fact"] { + return FactCovered(raw.(Obj), resolutionsOf(found)), nil + } + } + d.t.Fatalf("no fact %v", op["fact"]) + case "claimant": + found, _ := ReadAssignment(d.adir(op)) + for _, raw := range markerFactList(found, "claims") { + if fieldOf(raw.(Obj), "factId") == op["fact"] { + return Claimant(raw.(Obj)), nil + } + } + d.t.Fatalf("no fact %v", op["fact"]) + case "correlated": + found, _ := ReadAssignment(d.adir(op)) + return Correlated(found, op["session"], nil), nil + case "select": + workspace := d.work() + if w, ok := op["workspace"]; ok { + workspace = d.path(w) + } + directory, found, unreadable, err := SelectAssignment(d.root(), workspace, op["session"]) + if err != nil { + return nil, err + } + var name, facts any + if directory != "" { + name, facts = filepath.Base(directory), found + } + return map[string]any{"assignment": name, "facts": facts, "unreadable": unreadable}, nil + case "disposition": + return PublishDisposition(d.root(), d.work(), d.assignment(op), op["session"], op["turn"], op["outcome"].(string), t0) + case "read_disposition": + found, readable := ReadDisposition(d.adir(op), op["session"], op["turn"]) + return map[string]any{"found": found, "readable": readable}, nil + case "digest": + return FactDigest(fromJSON(op["payload"]).(Obj)), nil + case "named": + var out []any + for _, v := range op["values"].([]any) { + out = append(out, Named(fromJSON(v))) + } + return out, nil + case "same": + var out []any + for _, pair := range op["pairs"].([]any) { + p := pair.([]any) + out = append(out, SameIdentity(fromJSON(p[0]), fromJSON(p[1]))) + } + return out, nil + case "workspace_key": + return WorkspaceKey(d.path(op["workspace"])) + case "symlink": + mustDo(d.t, os.Symlink(d.path(op["to"]), d.path(op["link"]))) + return nil, nil + case "marker_root": + chosen, err := ResolveMarkerRoot(d.path(op["explicit"])) + if err != nil { + return nil, err + } + return chosen.Record(), nil + case "set_env": + setEnv(d.t, op["name"].(string), op["value"], d.tree) + return nil, nil + } + d.t.Fatalf("unknown op %v", op["op"]) + return nil, nil +} + +func (d *markerDriver) target(op markerOp) string { + if p, ok := op["path"].(string); ok { + return filepath.Join(d.adir(op), p) + } + return d.path(op["target"]) +} + +func opOr(op markerOp, key string, fallback any) any { + if v, ok := op[key]; ok { + return v + } + return fallback +} + +func noneIfEmpty(s string) any { + if s == "" { + return nil + } + return s +} + +// requireSameOps compares the two answer lists entry by entry, and returns Python's. +func requireSameOps(t *testing.T, ops []markerOp, python, golang []any) []any { + t.Helper() + if len(python) != len(golang) { + t.Fatalf("python answered %d ops, go %d", len(python), len(golang)) + } + for i := range ops { + p, g := normalizeJSON(t, python[i]), normalizeJSON(t, jsonable(golang[i])) + p, g = withoutPID(p), withoutPID(g) + if !reflect.DeepEqual(p, g) { + pb, _ := json.Marshal(p) + gb, _ := json.Marshal(g) + t.Errorf("op %d %v differs from Python\ngo: %s\npython: %s", i, ops[i], gb, pb) + } + } + return python +} + +// withoutPID replaces a conflict's loserProcess (the writer's own pid, str(os.getpid())) with one +// token: the two sides are different processes, and nothing else in the record may differ. +func withoutPID(v any) any { + switch x := v.(type) { + case map[string]any: + out := map[string]any{} + for k, e := range x { + if k == "loserProcess" { + if _, isText := e.(string); isText { + e = "" + } + } + out[k] = withoutPID(e) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = withoutPID(e) + } + return out + } + return v +} + +// ok is the value an op returned on the Python side (Python is the reference). +func ok(t *testing.T, answer any) any { + t.Helper() + m, isMap := answer.(map[string]any) + value, present := m["ok"] + if !isMap || !present { + t.Fatalf("op did not return: %v", answer) + } + return value +} + +func reasonOf(t *testing.T, answer any) string { + t.Helper() + m, _ := answer.(map[string]any) + reason, _ := m["reason"].(string) + if reason == "" { + t.Fatalf("op was not refused: %v", answer) + } + return reason +} + +// sameOps runs ops on both sides, requires every answer to be equal, and returns Python's. +func sameOps(t *testing.T, env map[string]any, ops ...markerOp) []any { + t.Helper() + python, golang := runMarkerOps(t, env, ops) + return requireSameOps(t, ops, python, golang) +} diff --git a/internal/relay/delivery/marker_test.go b/internal/relay/delivery/marker_test.go new file mode 100644 index 00000000..49d17019 --- /dev/null +++ b/internal/relay/delivery/marker_test.go @@ -0,0 +1,188 @@ +package delivery + +import ( + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +// test_marker.py properties MRK-1..MRK-8. Every answer is compared whole with the real +// marker.py run over the same tree (testdata/markerops.py). + +func TestMRK01_the_first_writer_wins_and_the_rest_are_told_they_lost(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "publish", "target": "/markers/a/intent.json", "payload": map[string]any{"one": 1}}, + markerOp{"op": "publish", "target": "/markers/a/intent.json", "payload": map[string]any{"two": 2}}, + markerOp{"op": "read_file", "target": "/markers/a/intent.json"}, + ) + if ok(t, answers[0]) != Published || ok(t, answers[1]) != Exists { + t.Fatalf("outcomes %v", answers) + } + // Eight concurrent writers: exactly one link() wins, and the survivor is a whole record. + target := filepath.Join(t.TempDir(), "a", "bound.json") + mustDo(t, os.MkdirAll(filepath.Dir(target), 0o700)) + var wg sync.WaitGroup + var mu sync.Mutex + start := make(chan struct{}) + outcomes := map[string]int{} + for i := range 8 { + wg.Add(1) + go func() { + defer wg.Done() + <-start + outcome, err := Publish(target, Obj{{Key: "writer", Value: int64(i)}}, "") + mu.Lock() + defer mu.Unlock() + if err != nil { + outcomes["error"]++ + return + } + outcomes[outcome]++ + }() + } + close(start) + wg.Wait() + if outcomes[Published] != 1 || outcomes[Exists] != 7 { + t.Fatalf("outcomes %v", outcomes) + } + data, err := os.ReadFile(target) + mustDo(t, err) + if !strings.Contains(string(data), `"writer":`) { + t.Fatalf("torn survivor %q", data) + } +} + +func TestMRK02_publication_is_atomic_and_an_orphan_temp_is_not_a_fact(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "publish", "target": "/markers/a/intent.json", "payload": map[string]any{"one": 1}}, + markerOp{"op": "publish", "target": "/markers/a/intent.json", "payload": map[string]any{"two": 2}}, + markerOp{"op": "listdir", "target": "/markers/a"}, + // A writer that died mid-write leaves a temp and no target: readers see absence. + markerOp{"op": "write_raw", "path": "attempts/.0.json.tmp.999.deadbeef", "text": "{}"}, + markerOp{"op": "facts"}, + // And a retry after an orphan temp of the same name still publishes. + markerOp{"op": "write_raw", "target": "/markers/b/.intent.json.tmp.1.abc", "text": "{}"}, + markerOp{"op": "publish", "target": "/markers/b/intent.json", "payload": map[string]any{"one": 1}}, + ) + if names := ok(t, answers[2]).([]any); len(names) != 1 || names[0] != "intent.json" { + t.Fatalf("temp left behind: %v", names) + } + facts := ok(t, answers[4]).(map[string]any) + if attempts := facts["facts"].(map[string]any)["attempts"].([]any); len(attempts) != 0 || len(facts["unreadable"].([]any)) != 0 { + t.Fatalf("orphan read as a fact: %v", facts) + } + if ok(t, answers[6]) != Published { + t.Fatal("retry after an orphan temp") + } +} + +func TestMRK03_the_fact_digest_reproduces_the_contract_vector_and_excludes_factid(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "digest", "payload": map[string]any{"factId": "conflicts/0", "at": "2026-01-01T00:06:00+00:00"}}, + markerOp{"op": "digest", "payload": map[string]any{"factId": "a", "x": 1}}, + markerOp{"op": "digest", "payload": map[string]any{"factId": "b", "x": 1}}, + markerOp{"op": "digest", "payload": map[string]any{"b": []any{1, "\u00e9", nil, true}, "a": map[string]any{"z": 1.5, "y": "q\""}}}, + ) + if ok(t, answers[0]) != "30250e28118e703a042e74d53844e078bbd318ae45a4479eac217c385a5c284a" || ok(t, answers[1]) != ok(t, answers[2]) { + t.Fatalf("digests %v", answers) + } +} + +func TestMRK04_nothing_names_nothing_and_two_unnamed_never_match(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "named", "values": []any{nil, "", " ", 3, []any{}, map[string]any{}, true, "a", " \u3000"}}, + markerOp{"op": "same", "pairs": []any{[]any{nil, nil}, []any{"", ""}, []any{" ", " "}, []any{"a", nil}, []any{"a", "a"}}}, + ) + named := ok(t, answers[0]).([]any) + same := ok(t, answers[1]).([]any) + for i, want := range []bool{false, false, false, false, false, false, false, true, false} { + if named[i] != want { + t.Fatalf("named %v", named) + } + } + for i, want := range []bool{false, false, false, false, true} { + if same[i] != want { + t.Fatalf("same %v", same) + } + } +} + +func TestMRK05_the_reader_assigns_factids_and_reports_what_it_could_not_read(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "publish", "path": "intent.json", "payload": map[string]any{"issueKey": "REL-1"}}, + markerOp{"op": "publish", "path": "attempts/0.json", "payload": map[string]any{"outcome": "accepted"}}, + markerOp{"op": "publish", "path": "claims/sess/claim.json", "payload": map[string]any{"sessionId": "sess"}}, + markerOp{"op": "facts"}, + // An unreadable fact is reported and never read as absent. + markerOp{"op": "write_raw", "dispatch": "second", "path": "intent.json", "text": "{not json"}, + markerOp{"op": "facts", "dispatch": "second"}, + // A fact that is not a record reaches the reader as the wrong shape. + markerOp{"op": "unlink", "path": "claims/sess/claim.json"}, + markerOp{"op": "write_raw", "path": "claims/sess/claim.json", "text": `"bare"`}, + markerOp{"op": "facts"}, + // Bytes that are not text are unreadable too, not a crash. + markerOp{"op": "write_raw", "dispatch": "third", "path": "attempts/0.json", "text": "\xff\xfe"}, + markerOp{"op": "facts", "dispatch": "third"}, + ) + read := ok(t, answers[3]).(map[string]any)["facts"].(map[string]any) + if read["intent"].(map[string]any)["factId"] != "intent" || read["attempts"].([]any)[0].(map[string]any)["factId"] != "attempts/0" || read["claims"].([]any)[0].(map[string]any)["factId"] != "claims/sess/claim.json" { + t.Fatalf("factIds %v", read) + } + if unreadable := ok(t, answers[5]).(map[string]any)["unreadable"].([]any); len(unreadable) != 1 || unreadable[0] != "intent" { + t.Fatalf("unreadable %v", unreadable) + } + if claims := ok(t, answers[8]).(map[string]any)["facts"].(map[string]any)["claims"].([]any); claims[0] != "bare" { + t.Fatalf("claims %v", claims) + } +} + +func TestMRK06_a_symlinked_workspace_reaches_the_same_assignment(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "symlink", "to": "/work", "link": "/alias"}, + markerOp{"op": "workspace_key", "workspace": "/alias"}, + markerOp{"op": "workspace_key", "workspace": "/work"}, + ) + if ok(t, answers[1]) != ok(t, answers[2]) { + t.Fatal("keys differ") + } +} + +func TestMRK07_a_disposition_is_read_where_the_stop_identity_derives(t *testing.T) { + answers := sameOps(t, nil, + markerOp{"op": "publish", "path": "dispositions/sess/turn-1.json", "payload": map[string]any{"sessionId": "sess", "turnId": "turn-1", "outcome": "interrupted"}}, + markerOp{"op": "read_disposition", "session": "sess", "turn": "turn-1"}, + markerOp{"op": "read_disposition", "session": "sess", "turn": "turn-2"}, + markerOp{"op": "read_disposition", "session": "sess", "turn": ""}, + markerOp{"op": "read_disposition", "session": "..", "turn": "turn-1"}, + ) + found := ok(t, answers[1]).(map[string]any) + if found["found"].(map[string]any)["outcome"] != "interrupted" || found["readable"] != true { + t.Fatalf("found %v", found) + } + for _, i := range []int{2, 3, 4} { + if a := ok(t, answers[i]).(map[string]any); a["found"] != nil || a["readable"] != true { + t.Fatalf("answer %d %v", i, a) + } + } +} + +func TestMRK08_the_marker_root_is_flag_then_env_then_xdg_then_home_and_never_the_state_dir(t *testing.T) { + answers := sameOps(t, map[string]any{MarkerEnv: nil, "XDG_STATE_HOME": nil, "HOME": "/home"}, + markerOp{"op": "marker_root", "explicit": "/explicit/./x/"}, + markerOp{"op": "set_env", "name": MarkerEnv, "value": "/from-env"}, + markerOp{"op": "marker_root"}, + markerOp{"op": "set_env", "name": MarkerEnv, "value": nil}, + markerOp{"op": "set_env", "name": "XDG_STATE_HOME", "value": "/xdg"}, + markerOp{"op": "marker_root"}, + markerOp{"op": "set_env", "name": "XDG_STATE_HOME", "value": nil}, + markerOp{"op": "marker_root"}, + ) + for i, source := range map[int]string{0: "flag", 2: "env", 5: "xdg", 7: "home"} { + record := ok(t, answers[i]).(map[string]any) + if record["source"] != source || strings.Contains(record["path"].(string), "codex-session-relay") { + t.Fatalf("answer %d %v", i, record) + } + } +} diff --git a/internal/relay/delivery/multiparent_test.go b/internal/relay/delivery/multiparent_test.go new file mode 100644 index 00000000..b41e99da --- /dev/null +++ b/internal/relay/delivery/multiparent_test.go @@ -0,0 +1,290 @@ +package delivery + +import ( + "context" + "fmt" + "math" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_multi_parent_isolation.py MPI-1..MPI-5. + +var projects = map[string][3]string{"a": {"/repo-a", "AAA-1", "linear://project-alpha"}, "b": {"/repo-b", "BBB-1", "linear://project-beta"}} + +func (f *fixture) twoParentAssignment(name string, events int) (string, []string) { + p := projects[name] + par, chi := "01parent-"+name, "01child-"+name + root := filepath.Join(f.root, name) + mustDo(f.t, os.MkdirAll(root, 0o755)) + rid := f.register(regOpts{issue: p[1], dispatchRequest: "dispatch-" + name, parent: par, parentCwd: p[0], child: chi, childRoot: root, turn: "turn-" + name, scopeRef: p[2], recipients: []string{par, chi}, parentOnlySettings: true}) + f.host.addThread(par) + f.host.addThread(chi) + var ids []string + for i := 0; i < events; i++ { + path := filepath.Join(root, fmt.Sprintf("out-%d.txt", i)) + mustDo(f.t, os.WriteFile(path, []byte(fmt.Sprintf("%s-%d", name, i)), 0o644)) + entries, err := store.BuildManifest([]string{path}, []string{root}) + mustDo(f.t, err) + revision, _ := store.ManifestRevision(entries) + attempt := i + 1 + event, _ := store.EventID(rid, 1, revision, "ready_for_review", "turn-"+name, &attempt) + payload := Obj{{Key: "eventId", Value: event}, {Key: "relationshipId", Value: rid}, {Key: "executionGeneration", Value: int64(1)}, {Key: "attempt", Value: int64(attempt)}, {Key: "revisionHash", Value: revision}, {Key: "outcome", Value: "ready_for_review"}, {Key: "producer", Value: "child"}, + {Key: "turnRef", Value: Obj{{Key: "threadId", Value: chi}, {Key: "turnId", Value: "turn-" + name}, {Key: "turnStatus", Value: "completed"}}}, {Key: "manifest", Value: []any{Obj{{Key: "path", Value: path}, {Key: "sha256", Value: entries[0].SHA256}, {Key: "bytes", Value: *entries[0].Bytes}}}}, {Key: "emittedAt", Value: f.clock.ISO()}} + _, err = f.accept(payload, store.AcceptOptions{}) + mustDo(f.t, err) + _, err = f.delivery.Enqueue(f.ctx, event, "", "") + mustDo(f.t, err) + ids = append(ids, event) + f.clock.Advance(1) + } + return rid, ids +} + +func (f *fixture) ackedTurn(name, event string) TurnInfo { + f.mustAttempt(event, at(f.clock.Now())) + f.clock.Advance(5) + return f.host.startTurn("01parent-"+name, "ack-"+name, "inProgress", "") +} + +// inParallel runs each call on its own Store over the same file, released at one instant. +func (f *fixture) inParallel(work map[string]func(*Ack) (Obj, error)) (map[string]Obj, map[string]error) { + path := filepath.Join(f.tree, "gostate", "relay.sqlite3") + start := make(chan struct{}) + var wg sync.WaitGroup + var mu sync.Mutex + results, errs := map[string]Obj{}, map[string]error{} + for name, call := range work { + s, err := store.Open(f.ctx, path, "") + mustDo(f.t, err) + f.t.Cleanup(func() { _ = s.Close() }) + d := NewService(s, f.clock) + a := NewAck(d) + a.Sync = VerdictSync(s, f.clock) + wg.Add(1) + go func() { + defer wg.Done() + <-start + r, err := call(a) + mu.Lock() + defer mu.Unlock() + if err != nil { + errs[name] = err + } else { + results[name] = r + } + }() + } + close(start) + wg.Wait() + return results, errs +} + +func TestMPI01_each_parent_keeps_its_scope_reference_and_project_key(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "mpi", "scope") + f := newFixture(t, tree) + a, _ := f.twoParentAssignment("a", 1) + b, _ := f.twoParentAssignment("b", 1) + ra, _ := LoadRelationship(f.ctx, f.store, a) + rb, _ := LoadRelationship(f.ctx, f.store, b) + requireSameJSON(t, "keys", []any{ProjectKey(ra), ProjectKey(rb)}, python.Out["keys"]) + if ra.ScopeRef != "linear://project-alpha" || rb.ScopeRef != "linear://project-beta" || ProjectKey(ra) != "/repo-a" { + t.Fatal("scope refs and project keys") + } + requireSameTables(t, f, python) +} + +func TestMPI02_two_parents_acknowledging_at_once_do_not_cross(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "mpi", "acks") + f := newFixture(t, tree) + a, ai := f.twoParentAssignment("a", 1) + b, bi := f.twoParentAssignment("b", 1) + turns := map[string]TurnInfo{"a": f.ackedTurn("a", ai[0]), "b": f.ackedTurn("b", bi[0])} + events := map[string]string{"a": ai[0], "b": bi[0]} + work := map[string]func(*Ack) (Obj, error){} + for _, name := range []string{"a", "b"} { + name := name + work[name] = func(ack *Ack) (Obj, error) { + e := events[name] + return ack.Acknowledge(context.Background(), e, turns[name].TurnID, AckProof(e, turns[name].TurnID), true, nil, f.host) + } + } + results, errs := f.inParallel(work) + if len(errs) != 0 || len(python.Out["errors"].(map[string]any)) != 0 { + t.Fatalf("errors go %v python %v", errs, python.Out["errors"]) + } + for name, rid := range map[string]string{"a": a, "b": b} { + if v, _ := get(results[name], "accepted"); v != true { + t.Fatalf("%s not accepted", name) + } + row := f.one("SELECT * FROM acks WHERE event_id = ?", events[name]) + if row.S("ack_turn_id") != "ack-"+name || f.row(events[name]).S("state") != Acknowledged || f.one("SELECT relationship_id FROM events WHERE event_id = ?", events[name]).S("relationship_id") != rid { + t.Fatalf("%s crossed", name) + } + } +} + +func TestMPI03_two_parents_ruling_needs_changes_at_once_open_one_generation_each(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "mpi", "verdicts") + f := newFixture(t, tree) + a, ai := f.twoParentAssignment("a", 1) + b, bi := f.twoParentAssignment("b", 1) + ack := NewAck(f.delivery) + events := map[string]string{"a": ai[0], "b": bi[0]} + for _, name := range []string{"a", "b"} { + turn := f.ackedTurn(name, events[name]) + _, err := ack.Acknowledge(f.ctx, events[name], turn.TurnID, AckProof(events[name], turn.TurnID), true, nil, f.host) + mustDo(t, err) + } + work := map[string]func(*Ack) (Obj, error){} + for _, name := range []string{"a", "b"} { + name := name + work[name] = func(ack *Ack) (Obj, error) { + return ack.RecordVerdict(context.Background(), events[name], "needs_changes", "verdict-"+name, nil, []any{finding("c1", "needs_changes", "fix it")}, nil, nil) + } + } + results, errs := f.inParallel(work) + if len(errs) != 0 { + t.Fatalf("errors %v", errs) + } + next := map[string]any{} + for name, rid := range map[string]string{"a": a, "b": b} { + n, _ := get(results[name], "nextExecutionGeneration") + next[name] = n + r, _ := LoadRelationship(f.ctx, f.store, rid) + revisions, err := all(f.ctx, f.store, "SELECT * FROM deliveries WHERE kind = ? AND relationship_id = ?", Revision, rid) + mustDo(t, err) + if r.Generation != 2 || len(revisions) != 1 || revisions[0].S("recipient_task_id") != "01child-"+name { + t.Fatalf("%s: generation %d revisions %v", name, r.Generation, revisions) + } + } + requireSameJSON(t, "next", next, python.Out["next"]) +} + +func TestMPI04_each_outbox_job_names_its_document_and_one_claim_cannot_complete_another(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "mpi", "outbox") + f := newFixture(t, tree) + ack := NewAck(f.delivery) + ack.Sync = VerdictSync(f.store, f.clock) + jobs := map[string]any{} + for _, tc := range []struct{ name, doc string }{{"a", "https://linear.app/example/document/project-alpha-0000"}, {"b", "https://linear.app/example/document/project-beta-00000"}} { + rid, ids := f.twoParentAssignment(tc.name, 1) + turn := f.ackedTurn(tc.name, ids[0]) + _, err := ack.Acknowledge(f.ctx, ids[0], turn.TurnID, AckProof(ids[0], turn.TurnID), true, nil, f.host) + mustDo(t, err) + _, err = execSQL(f.ctx, f.store, "INSERT INTO sync_targets (relationship_id, target, target_ref, recorded_at) VALUES (?,?,?,?) ON CONFLICT(relationship_id, target) DO UPDATE SET target_ref = excluded.target_ref, recorded_at = excluded.recorded_at", rid, coordinationDocument, tc.doc, f.clock.ISO()) + mustDo(t, err) + _, err = ack.RecordVerdict(f.ctx, ids[0], "verified", "verdict-"+tc.name, nil, nil, nil, nil) + mustDo(t, err) + jobs[tc.name] = f.one("SELECT sync_id FROM sync_outbox WHERE relationship_id = ?", rid).S("sync_id") + } + requireSameJSON(t, "jobs", jobs, python.Out["jobs"]) + alpha, err := SyncClaim(f.ctx, f.store, f.clock, jobs["a"].(string), "worker-1", f.clock.Now()) + mustDo(t, err) + _, err = SyncClaim(f.ctx, f.store, f.clock, jobs["b"].(string), "worker-1", f.clock.Now()) + mustDo(t, err) + _, err = SyncFenced(f.ctx, f.store, jobs["b"].(string), str(alpha, "claimToken")) + requireReason(t, err, SyncNotClaimable) + requireSameJSON(t, "complete", refusalOf(err), python.Out["complete"]) +} + +func (f *fixture) tick(sc *Scheduler) { + mustDo(f.t, sc.Deliver(f.ctx, f.host, f.clock.Now(), &TickCounts{})) +} + +func (f *fixture) drain(sc *Scheduler, ids []string) map[string]bool { + delivered := map[string]bool{} + for i := 0; i < 12 && len(delivered) < len(ids); i++ { + f.clock.Advance(3600) + f.tick(sc) + for _, e := range ids { + if f.row(e).S("state") == Dispatched { + delivered[e] = true + } + } + } + return delivered +} + +func TestMPI05_one_parents_limits_never_starve_the_other(t *testing.T) { + scheduler := func(f *fixture) *Scheduler { return &Scheduler{Delivery: f.delivery, Ack: NewAck(f.delivery)} } + t.Run("A at its attempt cap", func(t *testing.T) { + f := newFixture(t, "") + _, ai := f.twoParentAssignment("a", 1) + _, bi := f.twoParentAssignment("b", 3) + for i := int64(0); i < f.delivery.Policy.MaxAttempts; i++ { + f.host.script = []string{"read_fail"} + f.clock.Advance(100000) + f.mustAttempt(ai[0], at(f.clock.Now())) + } + if f.row(ai[0]).S("hold_reason") != AttemptCap { + t.Fatal("attempt_cap") + } + before := f.count("SELECT COUNT(*) AS c FROM attempts WHERE event_id = ?", ai[0]) + if len(f.drain(scheduler(f), bi)) != 3 || f.count("SELECT COUNT(*) AS c FROM attempts WHERE event_id = ?", ai[0]) != before { + t.Fatal("B drained and A's cap held") + } + }) + t.Run("A at the hourly recipient cap", func(t *testing.T) { + f := newFixture(t, "") + _, ai := f.twoParentAssignment("a", 1) + _, bi := f.twoParentAssignment("b", 3) + now := f.clock.Now() + _, err := execSQL(f.ctx, f.store, "INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,?,?)", "01parent-a", math.Floor(now/3600)*3600, f.delivery.Policy.MaxSendsPerRecipientPerHour, now) + mustDo(t, err) + sc := scheduler(f) + f.tick(sc) + sent := map[string]bool{} + for _, s := range f.host.sends { + sent[s.thread] = true + } + if sent["01parent-a"] || !sent["01parent-b"] || f.row(ai[0]).S("state") == Dispatched { + t.Fatalf("sent in the capped window: %v", sent) + } + if len(f.drain(sc, bi)) != 3 || f.row(ai[0]).S("state") != Dispatched { + t.Fatal("B drains; the cap is a delay, not a wall") + } + }) + t.Run("A paused, B served in that tick", func(t *testing.T) { + f := newFixture(t, "") + a, ai := f.twoParentAssignment("a", 1) + _, bi := f.twoParentAssignment("b", 2) + f.rid = a + f.setStatusBy("paused", "the owner of project alpha") + f.clock.Advance(3600) + before := len(f.host.sends) + f.tick(scheduler(f)) + sent := map[string]bool{} + for _, s := range f.host.sends[before:] { + sent[s.thread] = true + } + arrived := 0 + for _, e := range bi { + if f.row(e).S("state") == Dispatched { + arrived++ + } + } + if !sent["01parent-b"] || sent["01parent-a"] || arrived == 0 || f.row(ai[0]).S("state") == Dispatched { + t.Fatalf("sent %v arrived %d", sent, arrived) + } + }) + t.Run("A archived, the shared scheduler keeps serving B", func(t *testing.T) { + f := newFixture(t, "") + a, ai := f.twoParentAssignment("a", 1) + _, bi := f.twoParentAssignment("b", 2) + f.rid = a + f.setStatusBy("paused", "the owner of project alpha") + f.setStatusBy("archived", "the owner of project alpha") + if len(f.drain(scheduler(f), bi)) != 2 || f.row(ai[0]).S("state") == Dispatched { + t.Fatal("B drains, A is never delivered") + } + }) +} diff --git a/internal/relay/delivery/onrequest_adapter_test.go b/internal/relay/delivery/onrequest_adapter_test.go new file mode 100644 index 00000000..7083bc35 --- /dev/null +++ b/internal/relay/delivery/onrequest_adapter_test.go @@ -0,0 +1,147 @@ +package delivery + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ORD-6..ORD-9. Python drives its REAL adapter (the pinned bridge's guarded send over a fake +// RPC endpoint) and its real supervisor channel; Go asserts the delivery-layer decisions those +// paths rest on and compares them with Python's receipts: +// - the resume the guarded send transmits (ResumeParams / the settings-free form), +// - the verification between resume and turn/start (VerifyResume: refusal, findings, notes), +// - the classification of the receipt (Classify), and +// - the chronology rule a folded turn is judged by (certainlyBefore, turn_predates_send). +// The wire sequence of the adapter is todo 28's and the channel's record is todo 24's. + +func runOrdAdapter(t *testing.T, mode string) map[string]any { + t.Helper() + root := repoRoot(t) + script, _ := filepath.Abs("testdata/ordadapter.py") + home := t.TempDir() + cmd := exec.Command("uv", "run", "--no-sync", "python", script, t.TempDir(), mode) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(home, "state"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, "PYTHONPATH="+filepath.Join(root, "packages", "codex-session-relay", "src")+":"+filepath.Join(root, "packages", "codex-session-relay")) + out, err := cmd.Output() + if err != nil { + t.Fatalf("python %s: %v", mode, err) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + var got map[string]any + mustDo(t, json.Unmarshal([]byte(lines[len(lines)-1]), &got)) + return got +} + +func toObj(t *testing.T, v any) any { + raw, err := json.Marshal(v) + mustDo(t, err) + o, err := loads(string(raw)) + mustDo(t, err) + return o +} + +func guarded(t *testing.T, python map[string]any) (Obj, []any) { + settings := TaskSettings{Data: toObj(t, python["settings"]).(Obj), SettingsFreeResume: python["settingsFree"] == true} + resumed := toObj(t, python["resumed"]) + params := settings.ResumeParams("thread-1") + if settings.SettingsFreeResume { + params = Obj{{Key: "threadId", Value: "thread-1"}, {Key: "excludeTurns", Value: true}} + } + requireSameJSON(t, "resume params", []any{params}, python["resumes"]) + if _, present := get(params, "approvalPolicy"); present { + t.Fatal("the relay sent an approval policy") + } + rpcError, findings, notes := VerifyResume(settings, resumed, "idle") + receipt := Obj{{Key: "status", Value: Accepted}, {Key: "rpcError", Value: nil}, {Key: "settingsNotes", Value: nil}, {Key: "settingsFindings", Value: nil}, {Key: "error", Value: nil}, {Key: "statusBeforeResume", Value: "idle"}} + methods := []any{"thread/read", "thread/resume"} + if rpcError != nil { + receipt = set(set(set(set(receipt, "status", FailedStatus), "rpcError", rpcError), "settingsFindings", findings), "error", "thread/resume: "+str(rpcError, "message")) + } else { + methods = append(methods, "turn/start") + if len(notes) > 0 { + receipt = set(receipt, "settingsNotes", notes) + } + receipt = set(receipt, "turnId", "fake-turn-1") + } + requireSameJSON(t, "receipt", without(receipt, "turnId"), python["receipt"]) + requireSameJSON(t, "methods", methods, python["methods"]) + full := append(Obj(nil), receipt...) + full = set(full, "resumed", resumed) + return full, methods +} + +func TestORD06_an_on_request_thread_is_started_and_its_difference_noted(t *testing.T) { + for _, route := range []string{"transmitted", "settings_free"} { + t.Run(route, func(t *testing.T) { + python := runOrdAdapter(t, route) + receipt, _ := guarded(t, python) + if f := Classify(receipt); f.DeliveryState != Dispatched || f.DeliveryState != python["delivery_state"] { + t.Fatalf("classified %v", f) + } + }) + } +} + +func TestORD07_untrusted_stays_stored_not_woken(t *testing.T) { + python := runOrdAdapter(t, "untrusted") + receipt, methods := guarded(t, python) + if f := Classify(receipt); f.DeliveryState != InboxOnly || f.DeliveryState != python["delivery_state"] || len(methods) != 2 { + t.Fatalf("classified %v", f) + } +} + +func TestORD08_a_supervisor_push_follows_the_same_policy_rule(t *testing.T) { + // The channel's own record (withheld_pre_send carrying transportDeliveryState inbox_only) + // is todo 24's; what it rests on here is the transport classification of the two answers. + for _, tc := range []struct{ mode, policy, transport, channel string }{ + {"sup_on_request", "on-request", Dispatched, Dispatched}, + {"sup_untrusted", "untrusted", InboxOnly, WithheldPreSend}, + } { + t.Run(tc.mode, func(t *testing.T) { + python := runOrdAdapter(t, tc.mode) + record := python["record"].(map[string]any) + settings := TaskSettings{Data: loadsObj(rawSettings("/supervisor", "never")), SettingsFreeResume: true} + resumed := loadsObj(rawResume("/supervisor", tc.policy)) + rpcError, _, _ := VerifyResume(settings, resumed, "idle") + receipt := Obj{{Key: "status", Value: Accepted}, {Key: "resumed", Value: resumed}, {Key: "turnId", Value: "t"}} + if rpcError != nil { + receipt = Obj{{Key: "status", Value: FailedStatus}, {Key: "resumed", Value: resumed}, {Key: "rpcError", Value: rpcError}, {Key: "error", Value: "thread/resume: x"}} + } + state := Classify(receipt).DeliveryState + transport := record["deliveryState"] + if v, ok := record["transportDeliveryState"]; ok { + transport = v + } + if state != tc.transport || transport != tc.transport || python["state"] != tc.channel { + t.Fatalf("go %s python %v/%v", state, transport, python["state"]) + } + }) + } +} + +func rawResume(cwd, approval string) string { + return dumps(Obj{{Key: "approvalPolicy", Value: approval}, + {Key: "sandbox", Value: Obj{{Key: "type", Value: "workspaceWrite"}, {Key: "writableRoots", Value: []any{}}, {Key: "networkAccess", Value: false}, {Key: "excludeTmpdirEnvVar", Value: false}, {Key: "excludeSlashTmp", Value: false}}}, + {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}, {Key: "model", Value: "anthropic/claude-opus-5"}, {Key: "reasoningEffort", Value: "xhigh"}, {Key: "activePermissionProfile", Value: nil}, + {Key: "thread", Value: Obj{{Key: "id", Value: "t"}, {Key: "environments", Value: []any{Obj{{Key: "environmentId", Value: "local"}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: []any{cwd}}}}}}}}) +} + +// TurnPredatesSend is supervisorchannel.TURN_PREDATES_SEND, the readback answer for a turn that +// began before the send (the channel is todo 24's; the word and the chronology rule are shared). +const TurnPredatesSend = "turn_predates_send" + +func TestORD09_a_push_folded_into_a_running_turn_is_not_a_completion(t *testing.T) { + python := runOrdAdapter(t, "folded") + started := python["turnStartedAt"].(float64) + if !certainlyBefore(started, python["sentAt"].(string)) || python["verified"] != TurnPredatesSend { + t.Fatalf("the folded turn predates the send: python %v", python) + } + record := python["record"].(map[string]any) + if python["state"] != Dispatched || record["deliveryState"] != Dispatched || python["resent"] != float64(0) { + t.Fatalf("stays dispatched and sends nothing more: %v", python) + } +} diff --git a/internal/relay/delivery/onrequest_test.go b/internal/relay/delivery/onrequest_test.go new file mode 100644 index 00000000..eaef31cc --- /dev/null +++ b/internal/relay/delivery/onrequest_test.go @@ -0,0 +1,133 @@ +package delivery + +import ( + "testing" +) + +// test_on_request_delivery.py ORD-1..ORD-5 here; ORD-6..ORD-9 in onrequest_adapter_test.go. + +func runORD(t *testing.T, mode string, goSide func(f *fixture, out map[string]any)) { + tree := t.TempDir() + python := runPython(t, tree, "ord", mode) + f := newFixture(t, tree) + out := map[string]any{} + goSide(f, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + if mode != "record" { + requireSameTables(t, f, python) + } +} + +func usable(data string) map[string]any { + err := (&TaskSettings{Data: loadsObj(data)}).RequireUsable() + if err != nil { + return refusalOf(err) + } + return map[string]any{"ok": nil} +} + +func TestORD01_an_on_request_record_is_usable_and_untrusted_granular_or_missing_are_refused(t *testing.T) { + runORD(t, "record", func(f *fixture, out map[string]any) { + out["onRequest"] = usable(rawSettings("/parent", "on-request")) + var refused []any + for _, bad := range []string{ + rawSettings("/parent", "untrusted"), + replaceApproval(rawSettings("/parent", "X"), `{"granular": {}}`), + replaceApproval(rawSettings("/parent", "X"), `null`), + } { + r := usable(bad) + switch r["reason"] { + case UnsupportedApprovalPolicy, SettingsMistyped, SettingsIncomplete: + default: + t.Fatalf("refused with %v", r) + } + refused = append(refused, r) + } + out["refused"] = refused + out["resume"] = []any{(&TaskSettings{Data: loadsObj(rawSettings("/parent", "never"))}).ResumeParams("t-1"), (&TaskSettings{Data: loadsObj(rawSettings("/parent", "on-request"))}).ResumeParams("t-1")} + }) +} + +func replaceApproval(settings, raw string) string { + o := loadsObj(settings) + v, _ := loads(raw) + return dumps(set(o, "approvalPolicy", v)) +} + +func TestORD02_resume_params_never_carry_an_approval_policy(t *testing.T) { + for _, policy := range []string{"never", "on-request"} { + params := (&TaskSettings{Data: loadsObj(rawSettings("/parent", policy))}).ResumeParams("t-1") + if _, present := get(params, "approvalPolicy"); present { + t.Fatalf("%s: resume carries approvalPolicy", policy) + } + } + // The whole params object is compared with Python's in ORD-1's "resume" row. +} + +func TestORD03_an_on_request_parent_is_woken_once(t *testing.T) { + runORD(t, "woken", func(f *fixture, out map[string]any) { + event := f.queuedEvent(regOpts{parentSettings: rawSettings("/parent", "on-request")}) + f.host.threads[parent].approvalPolicy = "on-request" + record := f.mustAttempt(event, nil) + out["record"] = record + f.clock.Advance(100000) + out["eligible"] = []any{} + if str(record, "deliveryState") != Dispatched || str(record, "recipientApprovalPolicy") != "on-request" || len(f.host.sends) != 1 || len(f.eligible()) != 0 { + t.Fatalf("record %v", record) + } + }) +} + +func TestORD04_a_parent_waiting_on_its_approver_is_busy_then_woken_once(t *testing.T) { + runORD(t, "busy", func(f *fixture, out map[string]any) { + event := f.queuedEvent(regOpts{parentSettings: rawSettings("/parent", "on-request")}) + f.host.threads[parent].approvalPolicy = "on-request" + f.host.script = []string{"busy"} + out["busy"] = f.mustAttempt(event, nil) + out["record"] = f.mustAttempt(event, at(f.row(event).F("next_eligible_at"))) + started := 0 + for _, s := range f.host.sends { + if s.outcome == "accepted" { + started++ + } + } + if started != 1 { + t.Fatalf("%d turn/starts", started) + } + }) +} + +func TestORD05_a_folded_start_settles_once_across_restarts(t *testing.T) { + runORD(t, "folded", func(f *fixture, out map[string]any) { + event := f.queuedEvent(regOpts{parentSettings: rawSettings("/parent", "on-request")}) + f.host.threads[parent].approvalPolicy = "on-request" + existing := f.host.startTurn(parent, "", "inProgress", "") + f.host.script = []string{"steer_existing"} + out["record"] = f.mustAttempt(event, nil) + if f.one("SELECT 1 AS x FROM acks WHERE event_id = ?", event) != nil { + t.Fatal("dispatch alone is not an acknowledgement") + } + ack := NewAck(f.delivery) + var acks []any + for i := 0; i < 2; i++ { + r, err := ack.Acknowledge(f.ctx, event, existing.TurnID, AckProof(event, existing.TurnID), true, nil, f.host) + mustDo(t, err) + acks = append(acks, r) + } + out["acks"] = acks + f.host.finishTurn(parent, existing.TurnID, "completed") + for _, th := range f.host.threads { + th.status = "notLoaded" + } + restarted := NewService(f.store, f.clock) + f.clock.Advance(100000) + after, err := restarted.Attempt(f.ctx, event, f.host, at(f.clock.Now()), "") + mustDo(t, err) + out["after"] = after + if f.count("SELECT COUNT(*) AS c FROM acks WHERE event_id = ?", event) != 1 || len(f.host.sends) != 1 { + t.Fatal("settles once, sends once") + } + }) +} diff --git a/internal/relay/delivery/opengeneration_test.go b/internal/relay/delivery/opengeneration_test.go new file mode 100644 index 00000000..53eb0a9b --- /dev/null +++ b/internal/relay/delivery/opengeneration_test.go @@ -0,0 +1,30 @@ +package delivery + +import ( + "context" + "database/sql" + "testing" +) + +// Carried from the todo 25 checker: the verdict path composes OpenGenerationIn inside its own +// transaction, and a replay of the same dispatch request id must return its generation rather +// than open another. Compared with registry.open_generation_in over the same fixture: the three +// answers and every table row. +func Test21_OpenGenerationIn_replays_a_dispatch_request_rather_than_opening_another(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "ogi") + f := newFixture(t, tree) + f.queuedEvent(regOpts{}) + open := func(request string) int64 { + var number int64 + mustDo(t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + var err error + number, err = OpenGenerationIn(ctx, f.store, f.clock, f.rid, request, "needs_changes_revision", nil) + return err + })) + return number + } + got := map[string]any{"first": open("revision-x"), "replay": open("revision-x"), "next": open("revision-y")} + requireSameJSON(t, "open_generation_in answers", got, python.Out) + requireSameTables(t, f, python) +} diff --git a/internal/relay/delivery/policy.go b/internal/relay/delivery/policy.go new file mode 100644 index 00000000..653fadb7 --- /dev/null +++ b/internal/relay/delivery/policy.go @@ -0,0 +1,94 @@ +package delivery + +import "math" + +// Hold and pacing words (policy.py). +const ( + AttemptCap = "attempt_cap" + BusyCap = "busy_cap" + PushChannelClosed = "push_channel_closed" + SupersededHold = "superseded" + HostLostTurn = "host_lost_turn" + TurnCheckUndecided = "turn_check_undecided" + UnknownSendLost = "unknown_send_lost" + UnknownSendUndecided = "unknown_send_undecided" + UnknownSendHoldNamed = "unknown_send_hold_named" + MinSendInterval = "min_send_interval" + HourlyCap = "hourly_cap" + RateWindowSeconds = 3600.0 + RecipientUndeliverable = "recipient_undeliverable" +) + +// RetryPolicy is policy.RetryPolicy with its defaults. +type RetryPolicy struct { + BusyBase, BusyMax float64 + BusyMaxAttempts int64 + PresendBase, PresendMax float64 + MaxAttempts int64 + MinSendInterval float64 + MaxSendsPerRecipientPerHour int64 + LifecycleRecheck float64 + Lease float64 + MaxSendsPerParentPerTick int +} + +func DefaultPolicy() RetryPolicy { + return RetryPolicy{BusyBase: 15, BusyMax: 300, BusyMaxAttempts: 40, PresendBase: 30, PresendMax: 900, MaxAttempts: 6, + MinSendInterval: 5, MaxSendsPerRecipientPerHour: 12, LifecycleRecheck: 60, Lease: 300, MaxSendsPerParentPerTick: 2} +} + +func (p RetryPolicy) DelayFor(attemptNo int64, reason string) float64 { + base, ceiling := p.PresendBase, p.PresendMax + if reason == "busy" { + base, ceiling = p.BusyBase, p.BusyMax + } + return math.Min(ceiling, base*math.Pow(2, float64(max(0, attemptNo-1)))) +} + +func (p RetryPolicy) CapFor(reason string) int64 { + if reason == "busy" { + return p.BusyMaxAttempts + } + return p.MaxAttempts +} + +func (p RetryPolicy) CapReason(reason string) string { + if reason == "busy" { + return BusyCap + } + return AttemptCap +} + +// RateWindows is the hour window of now and the earliest window the gap reaches. +func (p RetryPolicy) RateWindows(now float64) (float64, float64) { + window := math.Floor(now/RateWindowSeconds) * RateWindowSeconds + reach := RateWindowSeconds * (1 + math.Floor(p.MinSendInterval/RateWindowSeconds)) + return window, window - reach +} + +// Pacing is policy.pacing: why a recipient may not be woken now, or nil. +func (p RetryPolicy) Pacing(now float64, sends int64, last *float64) Obj { + window, _ := p.RateWindows(now) + capacity := p.MaxSendsPerRecipientPerHour + var gapEnds *float64 + if last != nil { + g := *last + p.MinSendInterval + gapEnds = &g + } + reading := Obj{{Key: "sends", Value: sends}, {Key: "cap", Value: capacity}, {Key: "windowStart", Value: window}} + if sends >= capacity { + if capacity <= 0 { + return append(Obj{{Key: "reason", Value: HourlyCap}, {Key: "reopensAt", Value: nil}}, append(reading, + F{Key: "detail", Value: "a cap of zero refuses every send; only a changed policy reopens it"})...) + } + reopens := window + RateWindowSeconds + if gapEnds != nil && *gapEnds > reopens { + reopens = *gapEnds + } + return append(Obj{{Key: "reason", Value: HourlyCap}, {Key: "reopensAt", Value: reopens}}, reading...) + } + if gapEnds != nil && now < *gapEnds { + return append(Obj{{Key: "reason", Value: MinSendInterval}, {Key: "reopensAt", Value: *gapEnds}}, reading...) + } + return nil +} diff --git a/internal/relay/delivery/preview.go b/internal/relay/delivery/preview.go deleted file mode 100644 index a43f464e..00000000 --- a/internal/relay/delivery/preview.go +++ /dev/null @@ -1,443 +0,0 @@ -// Package delivery holds the parts of delivery.py that more than one relay command reads. -// It starts with preview_message, which `show --event E --message` needs (todo 20) and the -// sender will reuse (todo 21). Self-contained: it reads the store through store.Store and -// depends on nothing in internal/relay/cli. -package delivery - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" -) - -// Delivery kinds (delivery.py COMPLETION, REVISION, MERGE_TURN_GRANT). -const ( - Completion = "completion_event" - Revision = "revision_request" - MergeTurnGrant = "merge_turn_grant" -) - -// ManifestLines is MANIFEST_LINES: how many deliverables or findings a plain message lists. -const ManifestLines = 10 - -// ErrRendererNotPorted marks a preview whose bytes come from report.py's composed work-report -// renderer and required_for_candidate (todo 24) or from the merge-turn grant notice for grants -// mergeturn.py queues (todo 26); a preview is refused rather than rendered differently. -var ErrRendererNotPorted = errors.New("this preview needs the composed work-report renderer (todo 24) or the merge-turn grant notice (todo 26), which are not ported yet") - -// ErrNoDelivery is DeliveryService.get's refusal for an event with no delivery row. -var ErrNoDelivery = errors.New("no delivery queued") - -// Preview is DeliveryService.preview_message: what the NEXT attempt would say, never evidence of -// what any attempt did say. Reads use ctx, so a caller inside a transaction sees its writes. -func Preview(ctx context.Context, s *store.Store, event string) (string, error) { - row, err := s.One(ctx, "SELECT * FROM deliveries WHERE event_id = ?", event) - if err != nil { - return "", err - } - if row == nil { - return "", fmt.Errorf("%w for event %s", ErrNoDelivery, pyRepr(event)) - } - receipt, err := s.One(ctx, "SELECT * FROM events WHERE event_id = ?", event) - if err != nil { - return "", err - } - record := object{} - if receipt != nil { - if decoded, ok := decode(text(receipt.Get("receipt"))).(object); ok { - record = decoded - } - } - count, _ := receipt2int(row.Get("attempt_count")) - request, err := store.RequestID(event, int(count)+1) - if err != nil { - return "", fmt.Errorf("request id: %w", err) - } - report, err := s.One(ctx, "SELECT 1 FROM work_reports WHERE event_id = ? LIMIT 1", event) - if err != nil { - return "", err - } - kind := text(row.Get("kind")) - if report != nil || kind == MergeTurnGrant { - return "", ErrRendererNotPorted - } - if kind == Revision { - return renderRevision(event, text(row.Get("relationship_id")), record, request), nil - } - return renderCompletion(event, text(row.Get("relationship_id")), record, request), nil -} - -func receipt2int(v any) (int64, bool) { - n, ok := v.(int64) - return n, ok -} - -func text(v any) string { - switch value := v.(type) { - case string: - return value - case []byte: - return string(value) - } - return "" -} - -// renderCompletion is DeliveryService._render_completion with no work report. -func renderCompletion(event, relationship string, record object, request string) string { - lines := []string{ - "[codex-session-relay] verification request", - "requestId: " + request, - "eventId: " + event, - "relationshipId: " + relationship, - "executionGeneration: " + pyStr(record.get("executionGeneration")), - "attempt: " + pyStr(record.get("attempt")), - "outcome: " + pyStr(record.get("outcome")), - "revisionHash: " + pyStr(record.get("revisionHash")), - } - if manifest, ok := record.get("manifest").([]any); ok && len(manifest) > 0 { - lines = append(lines, fmt.Sprintf("deliverables: %d", len(manifest))) - for _, item := range manifest[:min(len(manifest), ManifestLines)] { - entry, _ := item.(object) - line := " " + pyStr(entry.get("path")) + " sha256=" + pyStr(entry.get("sha256")) - if size := entry.get("bytes"); size != nil { - line += " bytes=" + pyStr(size) - } - lines = append(lines, line) - } - if overflow := overflowLine(manifest, event, false); overflow != "" { - lines = append(lines, overflow) - } - } else { - lines = append(lines, "deliverables: none (execution-only outcome)") - } - if ref := record.get("manifestRef"); truthy(ref) { - lines = append(lines, "manifestRef: "+pyStr(ref)) - } - if criteria, ok := record.get("criteria").([]any); ok && len(criteria) > 0 { - lines = append(lines, "criteria claimed by the child:") - for _, item := range criteria[:min(len(criteria), ManifestLines)] { - entry, _ := item.(object) - lines = append(lines, " "+pyStr(entry.get("id"))+": "+pyStr(entry.get("verdict"))) - } - if overflow := overflowLine(criteria, event, false); overflow != "" { - lines = append(lines, overflow) - } - } - lines = append(lines, - "", - "To respond, from inside your own turn:", - " claim --event "+event+" --turn ", - " ack-proof --event "+event+" --turn ", - " ack --event "+event+" --ack-turn --ack-proof ", - " verdict --event "+event+" --verdict --verdict-turn ", - "", - "The proof is sha256(eventId|). This message does not and cannot", - "contain that turn id, which is what distinguishes acknowledging from echoing.", - "Full record: codex-session-relay show --event "+event, - ) - return strings.Join(lines, "\n") -} - -// renderRevision is DeliveryService._render_revision with no work report. -func renderRevision(event, relationship string, record object, request string) string { - lines := []string{ - "[codex-session-relay] revision request", - "requestId: " + request, - "eventId: " + event, - "relationshipId: " + relationship, - "executionGeneration: " + known(record.get("executionGeneration")) + " (new)", - "supersedesEvent: " + known(record.get("supersedesEvent")), - "supersedesRevisionHash: " + known(record.get("supersedesRevisionHash")), - "verdict: " + known(record.get("verdict")), - } - findings, _ := record.get("criteria").([]any) - lines = append(lines, "", violatedHeading(record)) - for _, item := range findings[:min(len(findings), ManifestLines)] { - finding, _ := item.(object) - id := finding.get("id") - if !truthy(id) { - id = "(no id recorded)" - } - verdict := finding.get("verdict") - if !truthy(verdict) { - verdict = "no disposition recorded" - } - line := " " + unheaded(inline(id)) + restorationLabel(finding) + ": " + inline(verdict) - if note := finding.get("note"); truthy(note) { - line += " \u2014 " + inline(note) - } else { - line += " \u2014 " + noNote - } - lines = append(lines, line) - } - if overflow := overflowLine(findings, event, true); overflow != "" { - lines = append(lines, inline(overflow)) - } - lines = append(lines, whatChangedLines(record)...) - lines = append(lines, fixScopeLines(record)...) - lines = append(lines, "", "PRESERVE: everything FIX SCOPE does not name, verified findings included,", - " work this request does not mention and any other task in-flight beside it") - lines = append(lines, reverifyLines(record)...) - lines = append(lines, returnLines(relationship, record.get("executionGeneration"))...) - lines = append(lines, "", "Full record: codex-session-relay show --event "+event) - return strings.Join(lines, "\n") -} - -// overflowLine is _overflow_line. -func overflowLine(items []any, event string, nameBlock bool) string { - if len(items) <= ManifestLines { - return "" - } - hidden := items[ManifestLines:] - detail := "" - if nameBlock { - for _, item := range hidden { - if finding, _ := item.(object); truthy(finding.get("restoration")) { - detail = ", including the restoration block on " + pyStr(finding.get("id")) - break - } - } - } - return fmt.Sprintf(" ... %d more%s; see 'codex-session-relay show --event %s'", len(hidden), detail, event) -} - -// restorationLabel is restoration.label. -func restorationLabel(finding object) string { - if truthy(finding.get("restoration")) { - return " [restoration block]" - } - return "" -} - -const ( - fix, evidenceOwed, met = "needs_changes", "unverified", "verified" - notRecorded = "not recorded" - noNote = "no note recorded; ask the parent" -) - -// headings is report.HEADINGS. -var headings = []string{"VIOLATED CRITERION", "WHAT CHANGED", "FIX SCOPE", "PRESERVE", "REVERIFY AND RETURN", - "TASK", "SCOPE", "MUST DO", "MUST NOT", "PROOF", "RETURN FORMAT", "DECISION BOUNDARY", "VERDICT"} - -// known is report.known. -func known(value any) string { - if value == nil || value == "" { - return notRecorded - } - return inline(value) -} - -// inline is report.inline: str.splitlines boundaries become " / ". -func inline(value any) string { - text := pyStr(value) - parts := splitlines(text) - if len(parts) <= 1 && (len(parts) == 0 || parts[0] == text) { - return text - } - var kept []string - for _, part := range parts { - if strings.TrimSpace(part) != "" && strings.TrimFunc(part, isPySpace) != "" { - kept = append(kept, part) - } - } - return strings.Join(kept, " / ") -} - -// splitlines is str.splitlines() without keepends. -func splitlines(text string) []string { - var parts []string - var current strings.Builder - runes := []rune(text) - for i := 0; i < len(runes); i++ { - switch r := runes[i]; r { - case '\n', '\v', '\f', 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029: - parts = append(parts, current.String()) - current.Reset() - case '\r': - parts = append(parts, current.String()) - current.Reset() - if i+1 < len(runes) && runes[i+1] == '\n' { - i++ - } - default: - current.WriteRune(r) - } - } - if current.Len() > 0 { - parts = append(parts, current.String()) - } - return parts -} - -// isPySpace is str.isspace for one character. -func isPySpace(r rune) bool { - switch r { - case ' ', '\t', '\n', '\v', '\f', '\r', 0x1c, 0x1d, 0x1e, 0x1f, 0x85, 0xa0, 0x1680, 0x2028, 0x2029, 0x202f, 0x205f, 0x3000: - return true - } - return r >= 0x2000 && r <= 0x200a -} - -// unheaded is report.unheaded. -func unheaded(value string) string { - probe := strings.TrimFunc(value, isPySpace) - probe = strings.TrimLeft(probe, "-*#>") - probe = strings.TrimFunc(probe, isPySpace) - probe = strings.ToUpper(strings.Trim(probe, "*`_")) - for _, name := range headings { - if probe == name || strings.HasPrefix(probe, name+":") { - return `"` + value + `"` - } - } - return value -} - -// correctionSource is report.correction_source with no review: the verdict's own findings. -func correctionSource(record object) []object { - var found []object - items, _ := record.get("criteria").([]any) - for _, item := range items { - if finding, ok := item.(object); ok && truthy(finding.get("id")) { - found = append(found, finding) - } - } - return found -} - -type counts struct{ fix, owed, met, undecided int } - -func countFindings(findings []object) counts { - var c counts - for _, finding := range findings { - switch finding.get("verdict") { - case fix: - c.fix++ - case evidenceOwed: - c.owed++ - case met: - c.met++ - default: - c.undecided++ - } - } - return c -} - -func plural(count int, noun string) string { - if count == 1 { - return fmt.Sprintf("%d %s", count, noun) - } - return fmt.Sprintf("%d %ss", count, noun) -} - -func notRecordedFor(what string) string { return notRecorded + ": " + what + "; ask the parent" } - -func series(words []string) string { - if len(words) == 1 { - return words[0] - } - return strings.Join(words[:len(words)-1], ", ") + " and " + words[len(words)-1] -} - -// violatedHeading is report.violated_heading with no review. -func violatedHeading(record object) string { - findings := correctionSource(record) - if len(findings) == 0 { - return "VIOLATED CRITERION: " + notRecordedFor("the verdict named no criterion") - } - c := countFindings(findings) - var parts []string - for _, part := range []struct { - n int - text string - }{{c.fix, "marked needs_changes"}, {c.owed, "marked unverified"}, {c.met, "marked verified"}, {c.undecided, "without a disposition"}} { - if part.n > 0 { - parts = append(parts, fmt.Sprintf("%d %s", part.n, part.text)) - } - } - return "VIOLATED CRITERION: of the " + plural(len(findings), "recorded finding") + ", " + strings.Join(parts, ", ") -} - -// whatChangedLines is report.what_changed_lines, plain, with no review. -func whatChangedLines(record object) []string { - return []string{"", - "WHAT CHANGED: submission " + known(record.get("supersedesEvent")) + " ruled " + known(record.get("verdict")) + - " and superseded, generation " + known(record.get("executionGeneration")) + " opened", - " superseded revision " + known(record.get("supersedesRevisionHash")) + ", ruled in turn " + known(record.get("verdictTurnId")), - } -} - -// fixScopeLines is report.fix_scope_lines, plain, with no review. -func fixScopeLines(record object) []string { - findings := correctionSource(record) - if len(findings) == 0 { - return []string{"", "FIX SCOPE: " + notRecordedFor("no criterion was named, so nothing bounds a change")} - } - c := countFindings(findings) - unruled := "FIX SCOPE: no finding is marked needs_changes, so nothing is ruled violated" - var heading string - switch { - case c.fix > 0: - heading = "FIX SCOPE: only the " + plural(c.fix, "finding") + " marked needs_changes, shown or not; " + - series([]string{"verified", "unverified"}) + " findings are out of scope" - case c.owed > 0: - heading = unruled + "; change nothing but the evidence REVERIFY AND RETURN asks for" - case c.undecided > 0: - heading = unruled + "; change nothing until the parent settles the findings below" - default: - heading = "FIX SCOPE: " + notRecordedFor("what to change, since every finding is marked verified") - } - lines := []string{"", heading} - if c.undecided > 0 { - pronoun := "them" - if c.undecided == 1 { - pronoun = "it" - } - lines = append(lines, " "+plural(c.undecided, "finding")+" without a disposition: whether to change "+pronoun+" is not recorded; ask the parent first") - } - return append(lines, " anything wider, anything that would discard preserved work, or anything needing authority you were not given comes back to the parent") -} - -// reverifyLines is report.reverify_lines with no review and no proof section. -func reverifyLines(record object) []string { - findings := correctionSource(record) - var clauses []string - if len(findings) == 0 { - clauses = []string{"what to re-check is " + notRecordedFor("the verdict named no criterion")} - } else { - c := countFindings(findings) - if c.fix > 0 { - clauses = append(clauses, "re-check each finding FIX SCOPE names") - } - if c.owed > 0 { - clauses = append(clauses, "show evidence for the "+plural(c.owed, "finding")+" marked unverified") - } - if c.undecided > 0 { - clauses = append(clauses, "settle with the parent whether to change the "+plural(c.undecided, "finding")+" without a disposition") - } - if len(clauses) == 0 { - clauses = append(clauses, "what to re-check is "+notRecordedFor("every finding is marked verified")) - } - } - return []string{"", "REVERIFY AND RETURN: " + strings.Join(append(clauses, "then hand back as below"), "; ")} -} - -// returnLines is report.return_lines. -func returnLines(relationship string, generation any) []string { - shown := "" - if generation != nil && generation != "" { - shown = inline(generation) - } - return []string{"", - "There is nothing to acknowledge. Contract v1 defines no acknowledgement for this", - "direction and the relay refuses one by kind, so there is no proof to compute and", - "no acknowledgement to send.", - "Answer with your next completion receipt under the new generation:", - " emit --relationship " + relationship + " --generation " + shown + " --attempt ", - " --outcome ready_for_review --turn-thread --turn-id ", - " --artifact [--continues-anchor ]", - } -} diff --git a/internal/relay/delivery/preview_test.go b/internal/relay/delivery/preview_test.go index 20709720..174c036a 100644 --- a/internal/relay/delivery/preview_test.go +++ b/internal/relay/delivery/preview_test.go @@ -5,7 +5,8 @@ import ( "testing" ) -// Values from the real report.inline / unheaded / known on this checkout's Python. +// Values from the real report.inline / unheaded / known on this checkout's Python (moved here +// with todo 20's preview when it was folded into this package's renderer). func TestText_helpers_match_python_report(t *testing.T) { for input, want := range map[string]string{"x": "x", "a\nb": "a / b", "a\r\nb\n": "a / b", "\n": "", "": "", "a\x1cb": "a / b", " \nz": "z", "a\u2028b": "a / b"} { diff --git a/internal/relay/delivery/projection.go b/internal/relay/delivery/projection.go new file mode 100644 index 00000000..bdb6de7d --- /dev/null +++ b/internal/relay/delivery/projection.go @@ -0,0 +1,123 @@ +package delivery + +import ( + "context" + "fmt" + "strings" +) + +// Anchored is assignment.AssignmentView._anchored for the fields this package owns: the event, +// its delivery and current attempt, the acknowledgement's two axes and the supersession note, +// read in one statement. The settings-hold reading and the undelivered reason of a withheld +// delivery are the assignment view's (todo 25); a delivery in such a state is refused here +// rather than reported without them. +func Anchored(ctx context.Context, d *Service, eventID any, generation int64) (Obj, error) { + record := Obj{{Key: "eventId", Value: eventID}, {Key: "executionGeneration", Value: generation}, {Key: "event", Value: nil}, {Key: "delivery", Value: nil}, {Key: "ack", Value: nil}, {Key: "undeliveredReason", Value: nil}, {Key: "supersession", Value: nil}} + if eventID == nil { + return append(record, F{Key: "detail", Value: "this generation has no such event"}), nil + } + now := d.Clock.Now() + window, earliest := d.Policy.RateWindows(now) + row, err := one(ctx, d.Store, `SELECT e.stage AS stage, d.event_id AS delivered, d.state AS delivery_state, d.hold_reason AS hold_reason, d.dispatch_evidence AS dispatch_evidence, d.next_eligible_at AS next_eligible_at, + a.request_id AS request_id, a.attempt_no AS attempt_no, a.state AS attempt_state, a.recipient_scan AS attempt_turn_check, v.last_reason AS ack_last_reason, + (SELECT COUNT(*) FROM attempts h WHERE h.event_id = e.event_id AND h.state = 'host_lost_turn') AS host_lost_attempts, + (SELECT sends FROM recipient_rate WHERE recipient_task_id = d.recipient_task_id AND window_start = ?) AS rate_sends, + (SELECT MAX(last_send_at) FROM recipient_rate WHERE recipient_task_id = d.recipient_task_id AND window_start BETWEEN ? AND ?) AS rate_last, + k.event_id AS acked, k.verified AS ack_verified, k.accepted AS ack_accepted, k.rejection_reason AS ack_rejection, v.tier AS ack_tier, + r.status AS relationship_status, r.superseded_by AS superseded_by, sx.reason AS supersession_reason, sx.applied AS supersession_applied, + (SELECT reason FROM refusals WHERE event_id = e.event_id ORDER BY id DESC LIMIT 1) AS refusal_reason + FROM events e LEFT JOIN deliveries d ON d.event_id = e.event_id LEFT JOIN attempts a ON a.event_id = e.event_id AND a.attempt_no = d.attempt_count + LEFT JOIN acks k ON k.event_id = e.event_id LEFT JOIN ack_evidence v ON v.event_id = e.event_id LEFT JOIN delivery_supersession sx ON sx.event_id = e.event_id + LEFT JOIN relationships r ON r.relationship_id = e.relationship_id WHERE e.event_id = ?`, window, earliest, window, eventID) + if err != nil { + return nil, err + } + if row == nil { + return append(record, F{Key: "detail", Value: "the store holds no such event"}), nil + } + record = set(record, "event", Obj{{Key: "stage", Value: row.S("stage")}}) + state := row.S("delivery_state") + delivered := !row.N("delivered") + if delivered { + if state == WithheldPreSend || state == InboxOnly { + return nil, fmt.Errorf("delivery: the settings-hold reading of a %s delivery is the assignment view's (todo 25)", state) + } + var turnCheck any + check := row.S("attempt_turn_check") + for _, prefix := range []string{"turn_check_undecided:", "unknown_send_lost:", "unknown_send_undecided:"} { + if strings.HasPrefix(check, prefix) { + turnCheck = check + } + } + var pacing any + if (state == Queued || state == DeferredBusy) && row.S("hold_reason") == "" { + var last *float64 + if !row.N("rate_last") { + v := row.F("rate_last") + last = &v + } + p := d.Policy.Pacing(now, row.I("rate_sends"), last) + if reopens, _ := get(p, "reopensAt"); p != nil && (reopens == nil || row.N("next_eligible_at") || row.F("next_eligible_at") <= reopens.(float64)) { + pacing = p + } + } + record = set(record, "delivery", Obj{{Key: "state", Value: state}, {Key: "requestId", Value: row.Opt("request_id")}, {Key: "attemptNo", Value: row.Opt("attempt_no")}, {Key: "attemptState", Value: row.Opt("attempt_state")}, + {Key: "dispatchEvidence", Value: row.Opt("dispatch_evidence")}, {Key: "holdReason", Value: row.Opt("hold_reason")}, {Key: "hostLostAttempts", Value: row.I("host_lost_attempts")}, {Key: "turnCheck", Value: turnCheck}, {Key: "pacing", Value: pacing}, {Key: "settingsHold", Value: nil}}) + } + acked := !row.N("acked") + pick := func(v any) any { + if acked { + return v + } + return nil + } + tier := any("unrecorded") + if !row.N("ack_tier") { + tier = row.S("ack_tier") + } + record = set(record, "ack", Obj{{Key: "accepted", Value: pick(row.I("ack_accepted") != 0)}, {Key: "rejectionReason", Value: pick(row.Opt("ack_rejection"))}, {Key: "settlement", Value: pick(row.Opt("ack_verified"))}, {Key: "lastReason", Value: pick(row.Opt("ack_last_reason"))}, {Key: "evidenceTier", Value: tier}}) + var undelivered any + switch { + case delivered && row.S("hold_reason") != "": + undelivered = Obj{{Key: "source", Value: "deliveries.hold_reason"}, {Key: "value", Value: row.S("hold_reason")}} + case delivered && (state == Queued || state == DeferredBusy) && !row.N("relationship_status") && row.S("relationship_status") != "active" && row.N("superseded_by"): + undelivered = Obj{{Key: "source", Value: "relationships.status"}, {Key: "value", Value: RelationshipNotActive}, {Key: "relationshipStatus", Value: row.S("relationship_status")}} + case !delivered && !row.N("refusal_reason"): + undelivered = Obj{{Key: "source", Value: "refusals.reason"}, {Key: "value", Value: row.S("refusal_reason")}} + } + record = set(record, "undeliveredReason", undelivered) + if !row.N("supersession_reason") { + record = set(record, "supersession", Obj{{Key: "reason", Value: row.S("supersession_reason")}, {Key: "applied", Value: row.I("supersession_applied") != 0}}) + } + return record, nil +} + +// Projection is the completion and correction anchors of the current generation. +func Projection(ctx context.Context, d *Service, rid string) (Obj, error) { + r, err := LoadRelationship(ctx, d.Store, rid) + if err != nil { + return nil, err + } + head, err := HeadRevision(ctx, d.Store, rid, r.Generation) + if err != nil { + return nil, err + } + headID, _ := get(head, "eventId") + completion, err := Anchored(ctx, d, headID, r.Generation) + if err != nil { + return nil, err + } + correction, err := one(ctx, d.Store, "SELECT event_id FROM events WHERE relationship_id = ? AND execution_generation = ? AND outcome = 'revision_request' AND suppressed_reason IS NULL ORDER BY event_id LIMIT 1", rid, r.Generation) + if err != nil { + return nil, err + } + var correctionID any + if correction != nil { + correctionID = correction.S("event_id") + } + anchored, err := Anchored(ctx, d, correctionID, r.Generation) + if err != nil { + return nil, err + } + return Obj{{Key: "completion", Value: completion}, {Key: "correction", Value: anchored}}, nil +} diff --git a/internal/relay/delivery/pyjson.go b/internal/relay/delivery/pyjson.go new file mode 100644 index 00000000..35db9d82 --- /dev/null +++ b/internal/relay/delivery/pyjson.go @@ -0,0 +1,243 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "unicode/utf16" + + "github.com/thisisjun786/codex-relay-workflow/internal/contract" +) + +// Obj is a Python dict in insertion order; the records this package returns keep Python's order. +type Obj = contract.OrderedObject + +// F is one field of an Obj. +type F = contract.Field + +// get returns the value of key in o, and whether it was present. +func get(o Obj, key string) (any, bool) { + for _, f := range o { + if f.Key == key { + return f.Value, true + } + } + return nil, false +} + +// set replaces key in o, or appends it, as a Python dict assignment does. +func set(o Obj, key string, value any) Obj { + for i, f := range o { + if f.Key == key { + o[i].Value = value + return o + } + } + return append(o, F{Key: key, Value: value}) +} + +func str(o Obj, key string) string { + v, _ := get(o, key) + s, _ := v.(string) + return s +} + +// dumps is Python json.dumps(value) with the default separators and ensure_ascii. +func dumps(value any) string { + var b strings.Builder + writeJSON(&b, value, false) + return b.String() +} + +// dumpsSorted is json.dumps(value, sort_keys=True). +func dumpsSorted(value any) string { + var b strings.Builder + writeJSON(&b, value, true) + return b.String() +} + +func writeJSON(b *strings.Builder, value any, sorted bool) { + switch v := value.(type) { + case nil: + b.WriteString("null") + case bool: + if v { + b.WriteString("true") + } else { + b.WriteString("false") + } + case string: + writeString(b, v) + case int: + b.WriteString(strconv.Itoa(v)) + case int64: + b.WriteString(strconv.FormatInt(v, 10)) + case float64: + b.WriteString(pyFloat(v)) + case json.Number: + b.WriteString(v.String()) + case Obj: + fields := v + if sorted { + fields = append(Obj(nil), v...) + sort.SliceStable(fields, func(i, j int) bool { return fields[i].Key < fields[j].Key }) + } + b.WriteByte('{') + for i, f := range fields { + if i > 0 { + b.WriteString(", ") + } + writeString(b, f.Key) + b.WriteString(": ") + writeJSON(b, f.Value, sorted) + } + b.WriteByte('}') + case []any: + b.WriteByte('[') + for i, item := range v { + if i > 0 { + b.WriteString(", ") + } + writeJSON(b, item, sorted) + } + b.WriteByte(']') + case []string: + items := make([]any, len(v)) + for i, s := range v { + items[i] = s + } + writeJSON(b, items, sorted) + default: + panic(fmt.Sprintf("delivery: unsupported JSON value %T", value)) + } +} + +func pyFloat(f float64) string { + switch { + case math.IsNaN(f): + return "NaN" + case math.IsInf(f, 1): + return "Infinity" + case math.IsInf(f, -1): + return "-Infinity" + } + m := math.Abs(f) + if m == 0 || m >= 1e-4 && m < 1e16 { + text := strconv.FormatFloat(f, 'f', -1, 64) + if !strings.Contains(text, ".") { + text += ".0" + } + return text + } + text := strconv.FormatFloat(f, 'e', -1, 64) + mantissa, exponent, _ := strings.Cut(text, "e") + digits := strings.TrimLeft(exponent[1:], "0") + if len(digits) < 2 { + digits = strings.Repeat("0", 2-len(digits)) + digits + } + return mantissa + "e" + exponent[:1] + digits +} + +func writeString(b *strings.Builder, s string) { + b.WriteByte('"') + for _, r := range s { + switch { + case r == '"' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r == '\n': + b.WriteString(`\n`) + case r == '\r': + b.WriteString(`\r`) + case r == '\t': + b.WriteString(`\t`) + case r == '\b': + b.WriteString(`\b`) + case r == '\f': + b.WriteString(`\f`) + case r < 0x20 || r > 0x7e: + for _, u := range utf16.Encode([]rune{r}) { + fmt.Fprintf(b, "\\u%04x", u) + } + default: + b.WriteRune(r) + } + } + b.WriteByte('"') +} + +// loads decodes JSON text into Obj/[]any/string/bool/nil/json.Number, keeping key order. +func loads(text string) (any, error) { + dec := json.NewDecoder(strings.NewReader(text)) + dec.UseNumber() + v, err := decodeValue(dec) + if err != nil { + return nil, err + } + if dec.More() { + return nil, fmt.Errorf("trailing data") + } + return v, nil +} + +func decodeValue(dec *json.Decoder) (any, error) { + tok, err := dec.Token() + if err != nil { + return nil, err + } + switch t := tok.(type) { + case json.Delim: + switch t { + case '{': + o := Obj{} + for dec.More() { + keyTok, err := dec.Token() + if err != nil { + return nil, err + } + key, _ := keyTok.(string) + v, err := decodeValue(dec) + if err != nil { + return nil, err + } + o = set(o, key, v) + } + if _, err := dec.Token(); err != nil { + return nil, err + } + return o, nil + case '[': + a := []any{} + for dec.More() { + v, err := decodeValue(dec) + if err != nil { + return nil, err + } + a = append(a, v) + } + if _, err := dec.Token(); err != nil { + return nil, err + } + return a, nil + } + case json.Number: + if i, err := t.Int64(); err == nil && !strings.ContainsAny(t.String(), ".eE") { + return i, nil + } + f, err := t.Float64() + return f, err + } + return tok, nil +} + +func loadsObj(text string) Obj { + v, err := loads(text) + if err != nil { + return nil + } + o, _ := v.(Obj) + return o +} diff --git a/internal/relay/delivery/qa_test.go b/internal/relay/delivery/qa_test.go new file mode 100644 index 00000000..32505cef --- /dev/null +++ b/internal/relay/delivery/qa_test.go @@ -0,0 +1,38 @@ +package delivery + +import ( + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Todo 21 QA (happy path): emit -> deliver -> claim -> ack -> verdict on a temp state dir, every +// row of the resulting store equal to the Python run of the same fixture. +func TestQA_emit_deliver_claim_ack_round_trip_rows_equal_python(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "qa") + f := newFixture(t, tree) + event := f.queuedEvent(regOpts{}) + delivered := f.mustAttempt(event, nil) + f.clock.Advance(5) + turn := f.host.startTurn(parent, "ack-turn", "inProgress", "") + ack := NewAck(f.delivery) + claim, err := ack.ClaimVerification(f.ctx, event, "ack-turn") + mustDo(t, err) + acked, err := ack.Acknowledge(f.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, f.host) + mustDo(t, err) + verdict, err := ack.RecordVerdict(f.ctx, event, "verified", "verdict-1", nil, nil, nil, nil) + mustDo(t, err) + payload := f.readyPayload(f.rid, 1, []string{f.artifact("out.txt", "the deliverable")}, 1, assigned("completed")) + stored, err := f.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + requireSameJSON(t, "delivered", delivered, python.Out["delivered"]) + requireSameJSON(t, "claim", claim, python.Out["claim"]) + requireSameJSON(t, "ack", acked, python.Out["ack"]) + requireSameJSON(t, "verdict", verdict, python.Out["verdict"]) + dup := python.Out["duplicate"].(map[string]any)["ok"].(map[string]any) + if !stored.Duplicate || dup["_duplicate"] != true { + t.Fatal("a re-emitted revision is a duplicate on both sides") + } + requireSameTables(t, f, python) +} diff --git a/internal/relay/delivery/race_recovery_test.go b/internal/relay/delivery/race_recovery_test.go new file mode 100644 index 00000000..25ba4064 --- /dev/null +++ b/internal/relay/delivery/race_recovery_test.go @@ -0,0 +1,249 @@ +package delivery + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_ack_disposition_race.py ADR-1..ADR-3 and test_recovery_negatives.py RCN-1..RCN-3, mirrored +// test for test (hostloss_harness_test.go mirror + testdata/capture.py). + +const adr = "test_ack_disposition_race" +const rcn = "test_recovery_negatives" + +// rival is CompetingAcknowledgements.rival: a second process, its own store connection. +func (h *hl) rival() *Ack { + h.t.Helper() + s, err := store.Open(context.Background(), h.store.Path, "") + mustDo(h.t, err) + h.t.Cleanup(func() { _ = s.Close() }) + return NewAck(NewService(s, h.clock)) +} + +type ackCall struct { + turn string + accepted bool + rejection any +} + +func (h *hl) ackWith(service *Ack, event string, call ackCall, adapter Adapter) Obj { + h.t.Helper() + out, err := service.Acknowledge(h.ctx, event, call.turn, AckProof(event, call.turn), call.accepted, call.rejection, adapter) + mustDo(h.t, err) + return out +} + +// racing commits the winner's acknowledgement inside the loser's pre-transaction host read and +// returns the adapter the loser uses plus what the winner got. +func (h *hl) racing(event string, winner *Ack, call ackCall) (Adapter, *bool, *Obj) { + raced, won := false, Obj(nil) + adapter := actsDuringTheTurnRead(h.host, func() { + raced = true + won = h.ackWith(winner, event, call, h.host) + }) + return adapter, &raced, &won +} + +func (h *hl) stored(event string) (Row, Obj) { + row := h.one("SELECT * FROM acks WHERE event_id = ?", event) + return row, loadsObj(row.S("record")) +} + +func (h *hl) raceDispatched() string { + event := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + h.attemptOn(event, h.host, nil) + h.clock.Advance(5) + return event +} + +func Test21_ADR01_a_verified_disposition_is_never_overwritten(t *testing.T) { + t.Run("losing rejection", func(t *testing.T) { + mirror(t, adr, "CompetingAcknowledgements.test_a_losing_rejection_does_not_overwrite_a_verified_acceptance", func(h *hl) { + event := h.raceDispatched() + accepting := h.host.startTurn(parent, "accepting-turn", "inProgress", "") + rejecting := h.host.startTurn(parent, "rejecting-turn", "inProgress", "") + adapter, raced, won := h.racing(event, h.rival(), ackCall{accepting.TurnID, true, nil}) + result := h.ackWith(h.ack, event, ackCall{rejecting.TurnID, false, "revision_mismatch"}, adapter) + h.eq(*raced) + h.eq(field(*won, "_verified")) + row, record := h.stored(event) + h.eq(truthy(field(record, "accepted"))) + h.eq(field(record, "rejectionReason")) + h.eq(row.S("ack_turn_id")) + h.eq(row.S("verified")) + h.eq(row.I("accepted")) + h.eq(truthy(field(result, "accepted"))) + h.eq(field(result, "ackTurnId")) + h.eq(truthy(field(result, "_replay"))) + h.eq(field(result, "_verified")) + h.eq(h.row(event).S("state")) + }) + }) + t.Run("losing acceptance", func(t *testing.T) { + mirror(t, adr, "CompetingAcknowledgements.test_a_losing_acceptance_does_not_overwrite_a_verified_rejection", func(h *hl) { + event := h.raceDispatched() + rejecting := h.host.startTurn(parent, "rejecting-turn", "inProgress", "") + accepting := h.host.startTurn(parent, "accepting-turn", "inProgress", "") + adapter, raced, _ := h.racing(event, h.rival(), ackCall{rejecting.TurnID, false, "revision_mismatch"}) + result := h.ackWith(h.ack, event, ackCall{accepting.TurnID, true, nil}, adapter) + h.eq(*raced) + row, record := h.stored(event) + h.eq(truthy(field(record, "accepted"))) + h.eq(field(record, "rejectionReason")) + h.eq(row.S("ack_turn_id")) + h.eq(row.S("verified")) + h.eq(truthy(field(result, "accepted"))) + h.eq(field(result, "ackTurnId")) + h.eq(truthy(field(result, "_replay"))) + }) + }) + t.Run("sequential", func(t *testing.T) { + mirror(t, adr, "CompetingAcknowledgements.test_a_sequential_second_disposition_is_unchanged", func(h *hl) { + event := h.raceDispatched() + accepting := h.host.startTurn(parent, "accepting-turn", "inProgress", "") + rejecting := h.host.startTurn(parent, "rejecting-turn", "inProgress", "") + h.ackWith(h.ack, event, ackCall{accepting.TurnID, true, nil}, h.host) + result := h.ackWith(h.ack, event, ackCall{rejecting.TurnID, false, "revision_mismatch"}, h.host) + h.eq(truthy(field(result, "accepted"))) + _, record := h.stored(event) + h.eq(truthy(field(record, "accepted"))) + h.eq(truthy(field(result, "_replay"))) + h.eq(field(result, "_verified")) + }) + }) +} + +func Test21_ADR02_a_losing_rejection_leaves_the_verdict_path_open(t *testing.T) { + mirror(t, adr, "CompetingAcknowledgements.test_the_losing_rejection_leaves_the_verdict_path_open", func(h *hl) { + event := h.raceDispatched() + accepting := h.host.startTurn(parent, "accepting-turn", "inProgress", "") + rejecting := h.host.startTurn(parent, "rejecting-turn", "inProgress", "") + adapter, _, _ := h.racing(event, h.rival(), ackCall{accepting.TurnID, true, nil}) + h.ackWith(h.ack, event, ackCall{rejecting.TurnID, false, "revision_mismatch"}, adapter) + record, err := h.verdict(event, "verified", "v1") + mustDo(t, err) + h.eq(field(record, "verdict")) + }) +} + +func Test21_ADR03_an_unverified_acknowledgement_is_still_upgradable(t *testing.T) { + mirror(t, adr, "CompetingAcknowledgements.test_an_unverified_acknowledgement_is_still_upgradable", func(h *hl) { + event := h.raceDispatched() + turn := h.host.startTurn(parent, "parent-own-turn", "inProgress", "") + h.ackWith(h.ack, event, ackCall{turn.TurnID, true, nil}, nil) + row, _ := h.stored(event) + h.eq(row.S("verified")) + results, err := h.ack.VerifyPendingAcks(h.ctx, h.host, 8, nil) + mustDo(t, err) + outcomes := []any{} + for _, r := range results { + outcomes = append(outcomes, str(r.(Obj), "outcome")) + } + h.eq(outcomes) + }) +} + +// ---------------------------------------------------------------- test_recovery_negatives.py + +func Test21_RCN01_a_restart_sweep_never_resends_or_retries(t *testing.T) { + mirror(t, rcn, "RecoveryRefusesToInvent.test_a_restart_sweep_never_resends", func(h *hl) { + event := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + h.attemptOn(event, h.host, nil) + report := h.recoverOnStart() + h.eq(field(report, "resent")) + h.eq(len(h.host.sends)) + }) + mirror(t, rcn, "RecoveryRefusesToInvent.test_an_uncertain_attempt_is_held_rather_than_retried", func(h *hl) { + event := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + h.host.script = []string{"process_death"} + h.attemptOn(event, h.host, nil) + state := h.one("SELECT state FROM deliveries WHERE event_id = ?", event).S("state") + h.eq(state == HeldUncertain || state == Queued || state == DeferredBusy) + h.eq(len(h.attemptsFor(event))) + }) +} + +func Test21_RCN02_reading_a_transcript_settles_nothing(t *testing.T) { + mirror(t, rcn, "RecoveryRefusesToInvent.test_reading_a_transcript_settles_nothing", func(h *hl) { + h.register(regOpts{recipients: []string{parent, child}}) + before := str(h.assignment(), "state") + events := h.count("SELECT COUNT(*) AS c FROM events") + _, err := h.host.ReadThread(child) + mustDo(t, err) + _, err = h.host.ListTurnIDs(child, 20) + mustDo(t, err) + after := str(h.assignment(), "state") + if after != before { + t.Fatalf("reading the transcript moved the assignment from %s to %s", before, after) + } + h.eq(after) + h.eq(h.count("SELECT COUNT(*) AS c FROM events")) + _ = events + }) +} + +// resumeRestating is registry._resume_in_transaction's restatement check (registry.resume is todo +// 25's; this is the refusing half, decided inside the write transaction, the same rows): a wrong +// restatement is refused relationship_not_active and writes nothing. +func (h *hl) resumeRestating(generation int64, roots, recipients []string, actor string) error { + return h.store.Transaction(h.ctx, func(ctx context.Context, _ *sql.Conn) error { + row, err := one(ctx, h.store, "SELECT * FROM relationships WHERE relationship_id = ?", h.rid) + if err != nil { + return err + } + var mismatches []string + if row.I("execution_generation") != generation { + mismatches = append(mismatches, fmt.Sprintf("generation is %d, not %d", row.I("execution_generation"), generation)) + } + var gotRoots, gotRecipients []string + mustDo(h.t, json.Unmarshal([]byte(row.S("artifact_roots")), &gotRoots)) + mustDo(h.t, json.Unmarshal([]byte(row.S("allowed_recipients")), &gotRecipients)) + if !reflect.DeepEqual(gotRoots, roots) { + mismatches = append(mismatches, "artifact roots differ from the restated scope") + } + if !reflect.DeepEqual(gotRecipients, recipients) { + mismatches = append(mismatches, "allowed recipients differ from the restated scope") + } + if truthy(row.Opt("superseded_by")) { + mismatches = append(mismatches, "superseded by "+row.S("superseded_by")) + } + if len(mismatches) > 0 { + return refuse(RelationshipNotActive, "resume refused: %s", strings.Join(mismatches, "; ")) + } + if _, err := execSQL(ctx, h.store, "UPDATE relationships SET status = ?, updated_at = ? WHERE relationship_id = ?", "active", h.clock.ISO(), h.rid); err != nil { + return err + } + return journal(ctx, h.store, "status_changed", h.rid, Obj{{Key: "status", Value: "active"}, {Key: "actor", Value: actor}}, h.clock.ISO()) + }) +} + +func Test21_RCN03_a_paused_assignment_is_never_auto_resumed(t *testing.T) { + mirror(t, rcn, "RecoveryRefusesToInvent.test_a_paused_assignment_is_never_auto_resumed", func(h *hl) { + h.register(regOpts{recipients: []string{parent, child}}) + h.setStatusBy("paused", parent) + h.recoverOnStart() + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(t, err) + h.eq(r.Status) + _, err = RequireActive(h.ctx, h.store, h.rid) + h.refusal(nil, err) + }) + mirror(t, rcn, "RecoveryRefusesToInvent.test_resuming_demands_a_restatement_rather_than_a_status_flip", func(h *hl) { + h.register(regOpts{recipients: []string{parent, child}}) + h.setStatusBy("paused", parent) + err := h.resumeRestating(99, []string{"/wrong"}, []string{parent}, parent) + if Reason(err) != RelationshipNotActive { + t.Fatalf("a wrong restatement was not refused: %v", err) + } + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(t, err) + h.eq(r.Status) + }) +} diff --git a/internal/relay/delivery/reconcile.go b/internal/relay/delivery/reconcile.go new file mode 100644 index 00000000..e764cf36 --- /dev/null +++ b/internal/relay/delivery/reconcile.go @@ -0,0 +1,649 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/mergeturn" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Affirmative evidence (reconcile.Evidence): the closed list; elapsed time is not on it. +const ( + TurnFound = "turn_found" + ReceiptTurnID = "receipt_turn_id" + ConfirmedPreSendRejection = "confirmed_pre_send_rejection" + NoEvidence = "none" + reconcileScanLimit = 200 + reconcileAction = "daemon_reconciles_delivery" + correctionUnconfirmed = "daemon_confirms_correction" + correctionHeld = "parent_recovers_held_correction" + correctionAnswered = "parent_reads_child_disposition" + unknownSendHeldAction = "parent_recovers_unknown_send_lost" + unknownSendUndecidedAction = "parent_recovers_unknown_send_undecided" + parentRecoveryThen = "read the report, then open a fresh execution generation (generation-open) if the work still needs verifying" +) + +// RelayProgram is the absolute command a rendered recovery line runs (relay_program): the crw +// binary under its compatibility name. +var RelayProgram = func() []string { + exe, err := os.Executable() + if err != nil { + return []string{"codex-session-relay"} + } + if real, err := filepath.EvalSymlinks(exe); err == nil { + exe = real + } + return []string{filepath.Join(filepath.Dir(exe), "codex-session-relay")} +} + +func shellQuote(s string) string { + if s == "" { + return "''" + } + safe := true + for _, r := range s { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("@%+=:,./-_", r)) { + safe = false + break + } + } + if safe { + return s + } + return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'" +} + +func recoveryCommand(stateDirectory, eventID string) string { + argv := append(RelayProgram(), "--state", stateDirectory, "show", "--event", eventID) + for i, a := range argv { + argv[i] = shellQuote(a) + } + return strings.Join(argv, " ") +} + +type attemptLost struct{} + +func (attemptLost) Error() string { return "attempt lost" } + +type attemptChanged struct{ moved bool } + +func (a attemptChanged) Error() string { return "attempt changed" } + +// Reconciler is reconcile.Reconciler: finding out what actually happened to an uncertain send. +type Reconciler struct { + Store *store.Store + Delivery *Service + Clock Clock +} + +func NewReconciler(d *Service) *Reconciler { + return &Reconciler{Store: d.Store, Delivery: d, Clock: d.Clock} +} + +func (rc *Reconciler) policy() RetryPolicy { return rc.Delivery.Policy } + +const unresolvedWhere = " WHERE (a.internal_state = 'in_flight' OR (a.state = ? AND d.state IN (?, ?)))" + +// OpenAttempts is open_attempts with no bound: everything a restart has to look at. +func (rc *Reconciler) OpenAttempts(ctx context.Context) ([]Row, error) { + return all(ctx, rc.Store, "SELECT a.*, r.parent_task_id AS parent_task_id FROM attempts a JOIN deliveries d ON d.event_id = a.event_id JOIN relationships r ON r.relationship_id = d.relationship_id"+unresolvedWhere+" ORDER BY a.observed_at", HeldUncertain, HeldUncertain, Sending) +} + +// ReconcileAttempt is reconcile_attempt. +func (rc *Reconciler) ReconcileAttempt(ctx context.Context, requestID string, adapter Adapter, now *float64) (Obj, error) { + out, err := rc.reconcile(ctx, requestID, adapter, now) + var changed attemptChanged + switch { + case errors.As(err, &attemptLost{}): + return rc.recordedLoss(ctx, requestID, nil) + case errors.As(err, &changed): + return rc.asItStands(ctx, requestID, changed, nil) + } + return out, err +} + +func (rc *Reconciler) asItStands(ctx context.Context, requestID string, changed attemptChanged, reading Reading) (Obj, error) { + attempt, err := one(ctx, rc.Store, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return nil, err + } + delivery, err := rc.Delivery.Get(ctx, attempt.S("event_id")) + if err != nil { + return nil, err + } + evidence := attempt.S("affirmative_evidence") + if evidence == "" { + evidence = NoEvidence + } + var record any + if !attempt.N("record") { + record = loadsObj(attempt.S("record")) + } + out := Obj{{Key: "evidence", Value: evidence}, {Key: "state", Value: attempt.Opt("state")}, {Key: "deliveryState", Value: delivery.S("state")}, {Key: "record", Value: record}} + if changed.moved { + out = append(out, F{Key: "changed", Value: true}, F{Key: "detail", Value: "another reader settled this attempt while this one read it; nothing was written, and the attempt as it now stands is reported"}) + } else { + out = append(out, F{Key: "kept", Value: true}, F{Key: "detail", Value: fmt.Sprintf("this attempt is already settled on %s; a reading that found no evidence does not write over it", pyStr(attempt.Opt("affirmative_evidence")))}) + } + if reading != nil { + out = append(out, F{Key: "recipientTurn", Value: reading}) + } + if attempt.S("state") == HeldUncertain && delivery.S("state") == HeldUncertain && delivery.I("attempt_count") == attempt.I("attempt_no") && evidence == NoEvidence { + aw, err := rc.awaiting(ctx, delivery.S("kind"), out, nil, Row{"hold_reason": delivery.Opt("hold_reason"), "recipient_scan": attempt.Opt("recipient_scan")}, attempt.S("event_id")) + if err != nil { + return nil, err + } + for _, f := range aw { + out = set(out, f.Key, f.Value) + } + } + return out, nil +} + +func (rc *Reconciler) recordedLoss(ctx context.Context, requestID string, reading Reading) (Obj, error) { + attempt, err := one(ctx, rc.Store, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return nil, err + } + rows, err := all(ctx, rc.Store, "SELECT detail FROM journal WHERE kind = ? AND subject = ? ORDER BY seq DESC", HostLostTurn, attempt.S("event_id")) + if err != nil { + return nil, err + } + var redelivery any + for _, row := range rows { + detail := loadsObj(row.S("detail")) + if str(detail, "requestId") == requestID { + redelivery, _ = get(detail, "redelivery") + break + } + } + evidence := attempt.S("affirmative_evidence") + if evidence == "" { + evidence = ReceiptTurnID + } + out := Obj{{Key: "evidence", Value: evidence}, {Key: "state", Value: HostLostTurn}, {Key: "redelivery", Value: redelivery}, {Key: "record", Value: loadsObj(attempt.S("record"))}, {Key: "detail", Value: "the host lost this attempt's turn and that is already recorded; nothing further was written"}} + if reading != nil { + out = append(out, F{Key: "recipientTurn", Value: reading}) + } + return out, nil +} + +func (rc *Reconciler) reconcile(ctx context.Context, requestID string, adapter Adapter, nowp *float64) (Obj, error) { + now := rc.Clock.Now() + if nowp != nil { + now = *nowp + } + attempt, err := one(ctx, rc.Store, "SELECT * FROM attempts WHERE request_id = ?", requestID) + if err != nil { + return nil, err + } + if attempt == nil { + return nil, fmt.Errorf("KeyError: %s", store.PyRepr(requestID)) + } + delivery, err := rc.Delivery.Get(ctx, attempt.S("event_id")) + if err != nil { + return nil, err + } + observation := "missing" + var facts *Facts + receiptRead := false + receipt, err := adapter.GetOperation(requestID) + if err != nil { + observation = "unreadable: " + errorLabel(err) + } else { + receiptRead = true + } + if receipt != nil { + f := Classify(receipt) + facts = &f + observation = f.ReceiptStatus + ":" + f.DeliveryState + if f.DeliveryState == Dispatched { + notes, _ := get(receipt, "settingsNotes") + return rc.settleDispatched(ctx, attempt, delivery, f, observation, adapter, now, notes) + } + if f.RetrySafe { + findings, _ := get(receipt, "settingsFindings") + return rc.settleFromReceipt(ctx, attempt, delivery, f, ConfirmedPreSendRejection, observation, now, false, nil, findings) + } + if f.ReceiptStatus != Unfinished { + observation += " (not affirmative)" + } + } + if attempt.S("state") == HeldUncertain && attempt.S("affirmative_evidence") == TurnFound { + return rc.settleConfirmed(ctx, attempt, delivery, observation, adapter) + } + scanDetail := "not scanned" + scanned := false + scan, err := adapter.FindToken(delivery.S("recipient_thread_id"), requestID, reconcileScanLimit, true) + if err != nil { + scanDetail = "unreadable: " + errorLabel(err) + } else { + scanDetail = fmt.Sprintf("found=%s exhausted=%s scanned=%d", pyStr(scan.Found), pyStr(scan.Exhausted), scan.Scanned) + if scan.Found { + return rc.settleFromScan(ctx, attempt, delivery, scan, observation, scanDetail, now) + } + scanned = true + } + var reading Reading + answer := "" + if receiptRead { + answer = receiptAnswer(facts) + } + if scanned && answer != "" { + reading = readUnknownSend(adapter, rc.Clock, attempt, delivery, answer) + if str(reading, "finding") == Present { + turn, _ := get(reading, "turnId") + out, err := rc.settleFromScan(ctx, attempt, delivery, TokenScan{Found: true, TurnID: turn}, observation, "found since the send in turn "+pyStr(turn), now) + if err != nil { + return nil, err + } + return append(out, F{Key: "recipientTrace", Value: reading}), nil + } + } + out, err := rc.stayHeld(ctx, attempt, delivery, observation, scanDetail, reading) + if err != nil { + return nil, err + } + if reading != nil { + out = append(out, F{Key: "recipientTrace", Value: reading}) + } + stored, err := one(ctx, rc.Store, "SELECT d.hold_reason, a.recipient_scan FROM attempts a JOIN deliveries d ON d.event_id = a.event_id WHERE a.request_id = ?", requestID) + if err != nil { + return nil, err + } + aw, err := rc.awaiting(ctx, delivery.S("kind"), out, reading, stored, attempt.S("event_id")) + if err != nil { + return nil, err + } + for _, f := range aw { + out = set(out, f.Key, f.Value) + } + return out, nil +} + +func receiptAnswer(facts *Facts) string { + if facts == nil { + return MissingReceipt + } + if facts.DeliveryState != HeldUncertain || facts.TurnID != nil || facts.RetrySafe { + return "" + } + if facts.ReceiptStatus == Unfinished { + return UnsettledReceipt + } + return SettledReceipt +} + +func (rc *Reconciler) settleDispatched(ctx context.Context, attempt, delivery Row, facts Facts, observation string, adapter Adapter, now float64, notes any) (Obj, error) { + reading := ReadRecipientTurn(adapter, rc.Clock, attempt, delivery, facts.TurnID) + out, err := rc.settleFromReceipt(ctx, attempt, delivery, facts, ReceiptTurnID, observation, now, str(reading, "finding") != Unknown, notes, nil) + var changed attemptChanged + switch { + case errors.As(err, &attemptLost{}): + return rc.recordedLoss(ctx, attempt.S("request_id"), reading) + case errors.As(err, &changed): + return rc.asItStands(ctx, attempt.S("request_id"), changed, reading) + case err != nil: + return nil, err + } + out = append(out, F{Key: "recipientTurn", Value: reading}) + if str(reading, "finding") == HostLostTurn { + if delivery.S("kind") == Completion { + loss, err := SettleLoss(ctx, rc.Store, rc.Clock, attempt.S("request_id"), reading, observation) + if err != nil { + return nil, err + } + for _, f := range loss { + out = set(out, f.Key, f.Value) + } + } else { + out = set(out, "redelivery", ReportOnly) + } + } else if delivery.S("kind") == Completion { + if _, err := RecordUndecided(ctx, rc.Store, attempt.S("request_id"), reading); err != nil { + return nil, err + } + } + return out, nil +} + +// readUnknownSend is ReadUnknownSend behind a test seam (Python's tests patch +// hostloss.read_unknown_send to stage a reader racing another); never reassigned in production. +var readUnknownSend = ReadUnknownSend + +// forceCurrent stands in for Python's mock.patch of _is_current (a test seam, never set in +// production): the guarded UPDATE, not the snapshot, must decide. +var forceCurrent bool + +func (rc *Reconciler) isCurrent(attempt, delivery Row) bool { + if forceCurrent { + return true + } + if attempt.I("attempt_no") != delivery.I("attempt_count") { + return false + } + s := delivery.S("state") + return s != Dispatched && s != Acknowledged && s != Superseded +} + +func (rc *Reconciler) settleFromReceipt(ctx context.Context, attempt, delivery Row, facts Facts, evidence, observation string, now float64, turnsChecked bool, notes, findings any) (Obj, error) { + record, err := AttemptRecord(facts, attempt.S("request_id"), attempt.S("event_id"), attempt.I("attempt_no"), delivery.S("recipient_task_id"), "unknown", rc.Clock.ISO(), + Obj{{Key: "operationReceiptChecked", Value: true}, {Key: "recipientTurnsChecked", Value: turnsChecked}, {Key: "affirmativeEvidence", Value: evidence}, {Key: "checkedAt", Value: rc.Clock.ISO()}}) + if err != nil { + return nil, err + } + var next, hold any + if facts.RetrySafe { + reason := "presend" + if facts.DeliveryState == DeferredBusy { + reason = "busy" + } + next = now + rc.policy().DelayFor(attempt.I("attempt_no")+1, reason) + if attempt.I("attempt_no") >= rc.policy().CapFor(reason) { + hold, next = rc.policy().CapReason(reason), nil + } + } + var dispatchEvidence any + if facts.DeliveryState == Dispatched { + dispatchEvidence = "transport_accepted" + } + anchor, err := rc.write(ctx, attempt, delivery, record, facts.DeliveryState, evidence, observation, "not scanned", next, writeOpts{hold: hold, dispatchEvidence: dispatchEvidence, dispatchTurn: facts.TurnID, settingsRefusal: settingsRefusalOf(facts, findings), notes: notes}) + if err != nil { + return nil, err + } + return withAnchor(Obj{{Key: "evidence", Value: evidence}, {Key: "state", Value: facts.DeliveryState}, {Key: "record", Value: record}}, anchor), nil +} + +func withAnchor(out Obj, anchor any) Obj { + if anchor != nil { + out = append(out, F{Key: "anchor", Value: anchor}) + } + return out +} + +func unfinishedRecord(attempt, delivery Row, observedAt string) Obj { + return Obj{{Key: "requestId", Value: attempt.S("request_id")}, {Key: "eventId", Value: attempt.S("event_id")}, {Key: "attemptNo", Value: attempt.I("attempt_no")}, {Key: "recipientTaskId", Value: delivery.S("recipient_task_id")}, + {Key: "deliveryState", Value: HeldUncertain}, {Key: "sendAttempted", Value: "unknown"}, {Key: "retrySafe", Value: false}, {Key: "transportReceiptStatus", Value: Unfinished}, {Key: "failedOperation", Value: nil}, {Key: "turnId", Value: nil}, + {Key: "recipientStatusBefore", Value: "unknown"}, {Key: "recipientApprovalPolicy", Value: nil}, {Key: "observedAt", Value: observedAt}} +} + +func (rc *Reconciler) settleFromScan(ctx context.Context, attempt, delivery Row, scan TokenScan, observation, scanDetail string, now float64) (Obj, error) { + var record Obj + if attempt.N("record") { + record = unfinishedRecord(attempt, delivery, rc.Clock.ISO()) + } else { + record = loadsObj(attempt.S("record")) + } + record = set(record, "reconciliation", Obj{{Key: "operationReceiptChecked", Value: true}, {Key: "recipientTurnsChecked", Value: true}, {Key: "affirmativeEvidence", Value: TurnFound}, {Key: "checkedAt", Value: rc.Clock.ISO()}}) + anchor, err := rc.write(ctx, attempt, delivery, record, str(record, "deliveryState"), TurnFound, observation, scanDetail, nil, writeOpts{aggregate: Dispatched, dispatchEvidence: "turn_found", dispatchTurn: scan.TurnID}) + if err != nil { + return nil, err + } + return withAnchor(Obj{{Key: "evidence", Value: TurnFound}, {Key: "state", Value: Dispatched}, {Key: "record", Value: record}}, anchor), nil +} + +func (rc *Reconciler) stayHeld(ctx context.Context, attempt, delivery Row, observation, scanDetail string, reading Reading) (Obj, error) { + var record Obj + if attempt.N("record") { + record = unfinishedRecord(attempt, delivery, rc.Clock.ISO()) + } else { + record = loadsObj(attempt.S("record")) + } + record = set(record, "reconciliation", Obj{{Key: "operationReceiptChecked", Value: true}, {Key: "recipientTurnsChecked", Value: !strings.Contains(scanDetail, "not scanned")}, {Key: "affirmativeEvidence", Value: NoEvidence}, {Key: "checkedAt", Value: rc.Clock.ISO()}}) + mark := scanDetail + var hold any + if reading != nil { + if u, _ := get(reading, "undecided"); truthy(u) { + mark, hold = unknownUndecided+pyStr(u), UnknownSendUndecided + } else if str(reading, "finding") == UnknownSendLost { + mark, hold = unknownLostMark, UnknownSendLost + } + } + if _, err := rc.write(ctx, attempt, delivery, record, HeldUncertain, NoEvidence, observation, mark, nil, writeOpts{hold: hold, keepUnknown: hold == nil, clearDispatch: true, expectScan: hold != nil}); err != nil { + return nil, err + } + return Obj{{Key: "evidence", Value: NoEvidence}, {Key: "state", Value: HeldUncertain}, {Key: "missing", Value: "no turn id in the operation receipt, no matching turn in the recipient's items, and no confirmed pre-send rejection"}, + {Key: "operationObservation", Value: observation}, {Key: "recipientScan", Value: scanDetail}, {Key: "record", Value: record}}, nil +} + +type writeOpts struct { + aggregate string + dispatchEvidence, dispatchTurn, hold any + keepUnknown, clearDispatch, expectScan bool + settingsRefusal, notes any +} + +func boolFlag(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func (rc *Reconciler) write(ctx context.Context, attempt, delivery Row, record Obj, state, evidence, observation, scanDetail string, next any, o writeOpts) (any, error) { + now := rc.Clock.ISO() + current := rc.isCurrent(attempt, delivery) + var anchor any + aggregate := o.aggregate + if aggregate == "" { + aggregate = state + } + err := rc.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + updated, err := execSQL(ctx, rc.Store, "UPDATE attempts SET internal_state = 'settled', state = ?, record = ?, operation_observation = ?, recipient_scan = CASE WHEN (? = ? AND recipient_scan LIKE ?) OR (? AND ? = ? AND recipient_scan LIKE ?) THEN recipient_scan ELSE ? END, affirmative_evidence = ?, reconciled_at = ? WHERE request_id = ? AND (state IS NULL OR state <> ?) AND internal_state IS ? AND state IS ? AND affirmative_evidence IS ? AND NOT (? = ? AND affirmative_evidence IS NOT NULL AND affirmative_evidence <> ?) AND (? = 0 OR recipient_scan IS ?)", + state, dumps(record), observation, state, Dispatched, TurnCheckUndecided+":%", boolFlag(o.keepUnknown), state, HeldUncertain, unknownMarkPrefix+"%", scanDetail, + evidence, now, attempt.S("request_id"), HostLostTurn, attempt.Opt("internal_state"), attempt.Opt("state"), attempt.Opt("affirmative_evidence"), + evidence, NoEvidence, NoEvidence, boolFlag(o.expectScan), attempt.Opt("recipient_scan")) + if err != nil { + return err + } + if updated != 1 { + now, err := one(ctx, rc.Store, "SELECT internal_state, state, affirmative_evidence, recipient_scan FROM attempts WHERE request_id = ?", attempt.S("request_id")) + if err != nil { + return err + } + if now != nil && now.S("state") == HostLostTurn { + return attemptLost{} + } + moved := now == nil || now.Opt("internal_state") != attempt.Opt("internal_state") || now.Opt("state") != attempt.Opt("state") || now.Opt("affirmative_evidence") != attempt.Opt("affirmative_evidence") || (o.expectScan && now.Opt("recipient_scan") != attempt.Opt("recipient_scan")) + return attemptChanged{moved: moved} + } + if current { + var before Row + if o.hold != nil { + if before, err = one(ctx, rc.Store, "SELECT hold_reason FROM deliveries WHERE event_id = ?", attempt.S("event_id")); err != nil { + return err + } + } + dispatchEvidence := o.dispatchEvidence + if dispatchEvidence == nil { + dispatchEvidence = delivery.Opt("dispatch_evidence") + } + promoted, err := execSQL(ctx, rc.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, hold_reason = CASE WHEN ? AND hold_reason IN (?, ?) THEN hold_reason ELSE ? END, dispatch_evidence = CASE WHEN ? THEN NULL ELSE ? END, dispatch_turn_id = CASE WHEN ? THEN NULL ELSE COALESCE(?, dispatch_turn_id) END, lease_owner = NULL, lease_until = NULL, updated_at = ? WHERE event_id = ? AND attempt_count = ? AND state NOT IN (?, 'acknowledged', 'superseded')", + aggregate, next, boolFlag(o.keepUnknown), UnknownSendLost, UnknownSendUndecided, o.hold, boolFlag(o.clearDispatch), dispatchEvidence, boolFlag(o.clearDispatch), o.dispatchTurn, now, attempt.S("event_id"), attempt.I("attempt_no"), Dispatched) + if err != nil { + return err + } + if promoted == 1 && aggregate == Dispatched { + if anchor, err = rc.bindPromotedAnchor(ctx, attempt, delivery, o.dispatchTurn); err != nil { + return err + } + } + var previous any + if before != nil { + previous = before.Opt("hold_reason") + } + if promoted == 1 && o.hold != nil && previous != o.hold { + if err := journal(ctx, rc.Store, UnknownSendHoldNamed, attempt.S("request_id"), Obj{{Key: "hold", Value: o.hold}, {Key: "previous", Value: previous}, {Key: "eventId", Value: attempt.S("event_id")}}, now); err != nil { + return err + } + } + } + if err := journal(ctx, rc.Store, "reconciled", attempt.S("request_id"), Obj{{Key: "evidence", Value: evidence}, {Key: "state", Value: aggregate}, {Key: "settingsRefusal", Value: o.settingsRefusal}}, now); err != nil { + return err + } + if list, _ := o.notes.([]any); len(list) > 0 { + noted, err := one(ctx, rc.Store, "SELECT 1 FROM journal WHERE subject = ? AND +kind = ? AND (CASE WHEN json_valid(detail) THEN json_extract(detail, '$.requestId') END) = ?", attempt.S("event_id"), SettingsNoted, attempt.S("request_id")) + if err != nil { + return err + } + if noted == nil { + return journal(ctx, rc.Store, SettingsNoted, attempt.S("event_id"), Obj{{Key: "requestId", Value: attempt.S("request_id")}, {Key: "notes", Value: o.notes}}, now) + } + } + return nil + }) + return anchor, err +} + +func (rc *Reconciler) bindPromotedAnchor(ctx context.Context, attempt, delivery Row, dispatchTurn any) (any, error) { + if delivery.S("kind") != Revision { + return nil, nil + } + turn, _ := dispatchTurn.(string) + if turn == "" { + turn = delivery.S("dispatch_turn_id") + } + if turn == "" { + return nil, nil + } + event, err := one(ctx, rc.Store, "SELECT relationship_id, execution_generation FROM events WHERE event_id = ?", attempt.S("event_id")) + if err != nil || event == nil { + return nil, err + } + return BindAnchorIn(ctx, rc.Store, rc.Clock, event.S("relationship_id"), event.I("execution_generation"), turn) +} + +// BindAnchorIn is registry.bind_anchor_in, inside the caller's transaction: bound, unchanged, +// conflict (journalled, never overwritten) or ineligible. +func BindAnchorIn(ctx context.Context, s *store.Store, clock Clock, rid string, number int64, turn string) (string, error) { + if strings.TrimSpace(turn) == "" { + return "ineligible", nil + } + current, err := one(ctx, s, "SELECT anchor_state, dispatch_turn_id FROM generations WHERE relationship_id = ? AND execution_generation = ?", rid, number) + if err != nil || current == nil { + return "ineligible", err + } + now := clock.ISO() + if current.S("anchor_state") == "bound" { + if current.S("dispatch_turn_id") == turn { + return "unchanged", nil + } + return "conflict", journal(ctx, s, "anchor_conflict", rid, Obj{{Key: "generation", Value: number}, {Key: "boundTo", Value: current.Opt("dispatch_turn_id")}, {Key: "offered", Value: turn}}, now) + } + if _, err := execSQL(ctx, s, "UPDATE generations SET anchor_state = ?, dispatch_turn_id = ?, bound_at = ? WHERE relationship_id = ? AND execution_generation = ?", "bound", turn, now, rid, number); err != nil { + return "", err + } + return "bound", journal(ctx, s, "anchor_bound", rid, Obj{{Key: "generation", Value: number}}, now) +} + +// awaiting is reconcile._awaiting for the kinds this package delivers. +func (rc *Reconciler) awaiting(ctx context.Context, kind string, outcome Obj, reading Reading, stored Row, eventID string) (Obj, error) { + if str(outcome, "state") != HeldUncertain { + return nil, nil + } + correction := kind == Revision + superseded := "" + if kind == MergeTurnGrant { + grant, err := rc.Delivery.SupersessionReason(ctx, eventID) + if err != nil { + return nil, err + } + if grant == mergeturn.GrantAnswered { + return Obj{{Key: "nextExpectedAction", Value: "none"}, {Key: "reason", Value: "grant_acknowledged"}}, nil + } + superseded = grant + } + if superseded == "" { + note, err := one(ctx, rc.Store, "SELECT reason FROM delivery_supersession WHERE event_id = ?", eventID) + if err != nil { + return nil, err + } + if note != nil { + superseded = note.S("reason") + } else if superseded, err = rc.Delivery.SupersessionReason(ctx, eventID); err != nil { + return nil, err + } + } + if superseded != "" { + action := "none" + if correction && superseded == SupersededRevision { + action = correctionAnswered + } + return Obj{{Key: "nextExpectedAction", Value: action}, {Key: "reason", Value: "superseded:" + superseded}}, nil + } + var hold, mark any + if stored != nil { + hold, mark = stored.Opt("hold_reason"), stored.Opt("recipient_scan") + } + var action string + var reason any + switch hold { + case UnknownSendLost: + action, reason = unknownSendHeldAction, UnknownSendLost + case UnknownSendUndecided: + action, reason = unknownSendUndecidedAction, mark + if !truthy(mark) { + reason = UnknownSendUndecided + } + default: + action = reconcileAction + if correction { + action = correctionUnconfirmed + } + reason, _ = get(outcome, "missing") + if reading != nil { + if d, _ := get(reading, "detail"); truthy(d) { + reason = d + } + } + return Obj{{Key: "nextExpectedAction", Value: action}, {Key: "reason", Value: reason}}, nil + } + if correction { + action = correctionHeld + } + directory := filepath.Dir(rc.Store.Path) + if abs, err := filepath.Abs(directory); err == nil { + directory = abs + } + return Obj{{Key: "nextExpectedAction", Value: action}, {Key: "reason", Value: reason}, + {Key: "recovery", Value: Obj{{Key: "actor", Value: "parent"}, {Key: "reason", Value: reason}, {Key: "command", Value: recoveryCommand(directory, eventID)}, {Key: "then", Value: parentRecoveryThen}}}}, nil +} + +// RecoverOnStart is recover_on_start: establish what happened; send nothing. +func (rc *Reconciler) RecoverOnStart(ctx context.Context, adapter Adapter, now *float64) (Obj, error) { + attempts, err := rc.OpenAttempts(ctx) + if err != nil { + return nil, err + } + reconciled, held, dispatched := []any{}, []any{}, []any{} + for _, a := range attempts { + outcome, err := rc.ReconcileAttempt(ctx, a.S("request_id"), adapter, now) + if err != nil { + return nil, err + } + entry := Obj{{Key: "requestId", Value: a.S("request_id")}} + for _, f := range outcome { + if f.Key != "record" { + entry = append(entry, f) + } + } + reconciled = append(reconciled, entry) + if str(outcome, "state") == HeldUncertain { + held = append(held, a.S("request_id")) + } + } + rows, err := all(ctx, rc.Store, "SELECT d.event_id FROM deliveries d LEFT JOIN acks a ON a.event_id = d.event_id WHERE d.state = ? AND d.kind = ? AND a.event_id IS NULL", Dispatched, Completion) + if err != nil { + return nil, err + } + for _, r := range rows { + dispatched = append(dispatched, r.S("event_id")) + } + return Obj{{Key: "reconciled", Value: reconciled}, {Key: "heldUncertain", Value: held}, {Key: "awaitingAck", Value: dispatched}, {Key: "resent", Value: []any{}}}, nil +} diff --git a/internal/relay/delivery/reconcile_pass.go b/internal/relay/delivery/reconcile_pass.go new file mode 100644 index 00000000..0152adbc --- /dev/null +++ b/internal/relay/delivery/reconcile_pass.go @@ -0,0 +1,244 @@ +package delivery + +import ( + "context" + "database/sql" + "strings" +) + +// The daemon's reconciliation pass (daemon._reconcile, _attempts_for, _gate, _mark_gate, +// _reads_were_complete) with the reconciler's bounded listings (open_attempts, open_parents, +// open_attempt_count). The pass is reconciliation's; the tick that runs it is todo 29's. + +// OpenAttemptsFor is open_attempts(limit, parents=[parent], offset). +func (rc *Reconciler) OpenAttemptsFor(ctx context.Context, parent string, limit, offset int) ([]Row, error) { + query := "SELECT a.*, r.parent_task_id AS parent_task_id FROM attempts a JOIN deliveries d ON d.event_id = a.event_id JOIN relationships r ON r.relationship_id = d.relationship_id" + unresolvedWhere + " AND r.parent_task_id IN (?) ORDER BY a.observed_at LIMIT ?" + args := []any{HeldUncertain, HeldUncertain, Sending, parent, limit} + if offset > 0 { + query += " OFFSET ?" + args = append(args, offset) + } + return all(ctx, rc.Store, query, args...) +} + +// OpenAttemptCount is open_attempt_count. +func (rc *Reconciler) OpenAttemptCount(ctx context.Context, parent string) (int, error) { + row, err := one(ctx, rc.Store, "SELECT COUNT(*) AS c FROM attempts a JOIN deliveries d ON d.event_id = a.event_id JOIN relationships r ON r.relationship_id = d.relationship_id"+unresolvedWhere+" AND r.parent_task_id = ?", HeldUncertain, HeldUncertain, Sending, parent) + if err != nil || row == nil { + return 0, err + } + return int(row.I("c")), nil +} + +// OpenParents is open_parents. +func (rc *Reconciler) OpenParents(ctx context.Context) ([]string, error) { + rows, err := all(ctx, rc.Store, "SELECT DISTINCT r.parent_task_id AS parent_task_id FROM attempts a JOIN deliveries d ON d.event_id = a.event_id JOIN relationships r ON r.relationship_id = d.relationship_id"+unresolvedWhere+" ORDER BY r.parent_task_id", HeldUncertain, HeldUncertain, Sending) + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.S("parent_task_id")) + } + return out, err +} + +// ReconcileReport is the reconciliation half of the daemon's TickReport. +type ReconcileReport struct { + Reconciled, Skipped int + Notes []string +} + +// ReconcilePass is daemon._reconcile: a budget dealt across parents from rotating cursors, each +// attempt reconciled only when its gate says there is something new to learn. +func ReconcilePass(ctx context.Context, rc *Reconciler, adapter Adapter, budget int, now float64, report *ReconcileReport) error { + if budget == 0 { + budget = 8 + } + parents, err := rc.OpenParents(ctx) + if err != nil || len(parents) == 0 { + return err + } + cursors := &Scheduler{Delivery: rc.Delivery} + cursor, err := cursors.cursor(ctx, "reconcile_parents", len(parents)) + if err != nil { + return err + } + order := append(append([]string(nil), parents[cursor:]...), parents[:cursor]...) + if err := cursors.advance(ctx, "reconcile_parents", 1, len(parents)); err != nil { + return err + } + share := max(1, budget/len(order)) + queues := make([][]Row, len(order)) + for i, parent := range order { + if queues[i], err = attemptsFor(ctx, rc, cursors, parent, share); err != nil { + return err + } + } + var dealt []Row + for len(dealt) < budget { + any := false + for i := range queues { + if len(dealt) >= budget { + break + } + if len(queues[i]) > 0 { + dealt = append(dealt, queues[i][0]) + queues[i] = queues[i][1:] + any = true + } + } + if !any { + break + } + } + for _, parent := range order { + taken := 0 + for _, r := range dealt { + if r.S("parent_task_id") == parent { + taken++ + } + } + if taken > 0 { + total, err := rc.OpenAttemptCount(ctx, parent) + if err != nil { + return err + } + if err := cursors.advance(ctx, "reconcile:"+parent, taken, total); err != nil { + return err + } + } + } + for _, attempt := range dealt { + id := attempt.S("request_id") + decision, fingerprint, err := gate(ctx, rc, adapter, attempt) + if err != nil { + return err + } + if !decision { + report.Skipped++ + continue + } + outcome, err := rc.ReconcileAttempt(ctx, id, adapter, &now) + if err != nil { + text := err.Error() + if err := markGate(ctx, rc, id, nil, true, text); err != nil { + return err + } + report.Notes = append(report.Notes, "reconcile failed for "+id+": "+text) + continue + } + complete := readsWereComplete(outcome) + var print any + var failure any + if complete { + print = fingerprint + } else { + failure = "reads incomplete" + } + if err := markGate(ctx, rc, id, print, !complete, failure); err != nil { + return err + } + report.Reconciled++ + } + return nil +} + +func attemptsFor(ctx context.Context, rc *Reconciler, cursors *Scheduler, parent string, share int) ([]Row, error) { + total, err := rc.OpenAttemptCount(ctx, parent) + if err != nil || total == 0 { + return nil, err + } + want := min(share, total) + start, err := cursors.cursor(ctx, "reconcile:"+parent, total) + if err != nil { + return nil, err + } + taken, err := rc.OpenAttemptsFor(ctx, parent, want, start) + if err != nil { + return nil, err + } + if len(taken) < want { + seen := map[string]bool{} + for _, r := range taken { + seen[r.S("request_id")] = true + } + again, err := rc.OpenAttemptsFor(ctx, parent, want, 0) + if err != nil { + return nil, err + } + for _, r := range again { + if len(taken) >= want { + break + } + if !seen[r.S("request_id")] { + taken = append(taken, r) + } + } + } + return taken, nil +} + +func readsWereComplete(outcome Obj) bool { + if v, _ := get(outcome, "changed"); truthy(v) { + return false + } + if trace, ok := get(outcome, "recipientTrace"); ok { + if o, _ := trace.(Obj); o != nil { + if p, _ := get(o, "pending"); truthy(p) { + return false + } + } + } + observation, scan := str(outcome, "operationObservation"), str(outcome, "recipientScan") + if strings.Contains(observation, "unreadable") || strings.Contains(scan, "unreadable") { + return false + } + if strings.HasPrefix(scan, "not scanned") { + return str(outcome, "evidence") != NoEvidence + } + return true +} + +// gate is daemon._gate: is there anything new to learn? The fingerprint is read first. +func gate(ctx context.Context, rc *Reconciler, adapter Adapter, attempt Row) (bool, any, error) { + id := attempt.S("request_id") + row, err := one(ctx, rc.Store, "SELECT * FROM reconcile_gate WHERE request_id = ?", id) + if err != nil { + return false, nil, err + } + delivery, err := rc.Delivery.Find(ctx, attempt.S("event_id")) + if err != nil { + return false, nil, err + } + receipt, readErr := adapter.GetOperation(id) + var content string + if readErr == nil { + content, readErr = adapter.RecipientFingerprint(delivery.S("recipient_thread_id")) + } + if readErr != nil { + return true, nil, markGate(ctx, rc, id, nil, true, readErr.Error()) + } + status := "missing" + if receipt != nil { + if v, ok := get(receipt, "status"); ok { + status = pyStr(v) + } + } + fingerprint := status + "|" + content + if row == nil || row.I("retry_required") != 0 { + return true, fingerprint, nil + } + if strings.HasPrefix(attempt.S("recipient_scan"), unknownMarkPrefix) { + updated, ok := epoch(row.S("updated_at")) + if !ok || rc.Clock.Now()-updated >= UndecidedRecheckSeconds { + return true, fingerprint, nil + } + } + return fingerprint != row.S("fingerprint"), fingerprint, nil +} + +func markGate(ctx context.Context, rc *Reconciler, requestID string, fingerprint any, retry bool, failure any) error { + return rc.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := execSQL(ctx, rc.Store, "INSERT INTO reconcile_gate (request_id, fingerprint, retry_required, last_error, updated_at) VALUES (?,?,?,?,?) ON CONFLICT(request_id) DO UPDATE SET fingerprint=COALESCE(excluded.fingerprint, reconcile_gate.fingerprint), retry_required=excluded.retry_required, last_error=excluded.last_error, updated_at=excluded.updated_at", + requestID, fingerprint, boolFlag(retry), failure, rc.Clock.ISO()) + return err + }) +} diff --git a/internal/relay/delivery/registration_hold_live_test.go b/internal/relay/delivery/registration_hold_live_test.go new file mode 100644 index 00000000..22ebf014 --- /dev/null +++ b/internal/relay/delivery/registration_hold_live_test.go @@ -0,0 +1,50 @@ +package delivery + +import ( + "context" + "database/sql" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" + _ "modernc.org/sqlite" +) + +func Test21_RegistrationHold_refuses_resolved_live_state(t *testing.T) { + root := t.TempDir() + live := filepath.Join(root, "xdg", "codex-session-relay", "default") + if err := os.MkdirAll(live, 0700); err != nil { + t.Fatal(err) + } + dbPath := filepath.Join(live, "relay.sqlite3") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatal(err) + } + if _, err := db.Exec("CREATE TABLE probe (id INTEGER)"); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + alias := filepath.Join(root, "alias.sqlite3") + // An outside alias resolves into a temp directory shaped like live state. + if err := os.Symlink(dbPath, alias); err != nil { + t.Fatal(err) + } + t.Setenv("XDG_STATE_HOME", filepath.Join(root, "xdg")) + t.Setenv("CRW_ALLOW_LIVE_STATE", "") + called := false + err = store.RegistrationHold(context.Background(), alias, func(conn *sql.Conn, why string) error { + called = true + if conn != nil || !strings.Contains(why, store.ErrLiveState.Error()) { + t.Errorf("hold conn=%v why=%q", conn, why) + } + return nil + }) + if err != nil || !called { + t.Fatalf("hold called=%v err=%v", called, err) + } +} diff --git a/internal/relay/delivery/relationship.go b/internal/relay/delivery/relationship.go new file mode 100644 index 00000000..eab4d2fd --- /dev/null +++ b/internal/relay/delivery/relationship.go @@ -0,0 +1,92 @@ +package delivery + +import ( + "context" + "encoding/json" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Relationship is registry.get's record, read with the generation invariant enforced. +type Relationship struct { + ID string + IssueKey string + Status string + Parent, Child Endpoint + Generation int64 + ArtifactRoots []string + AllowedRecipients []string + ScopeRef any + SupersededBy any + Generations []Row +} + +// Endpoint is one side of an assignment. +type Endpoint struct { + TaskID, HostID string + Cwd any +} + +// Active is status active and not superseded. +func (r Relationship) Active() bool { return r.Status == "active" && r.SupersededBy == nil } + +func (r Relationship) generation(number int64) Row { + for _, g := range r.Generations { + if g.I("execution_generation") == number { + return g + } + } + return nil +} + +// LoadRelationship is registry.get. +func LoadRelationship(ctx context.Context, s *store.Store, id string) (Relationship, error) { + row, err := one(ctx, s, "SELECT * FROM relationships WHERE relationship_id = ?", id) + if err != nil { + return Relationship{}, err + } + if row == nil { + return Relationship{}, refuse(UnregisteredRelationship, "no relationship %s", store.PyRepr(id)) + } + generations, err := all(ctx, s, "SELECT * FROM generations WHERE relationship_id = ? ORDER BY execution_generation", id) + if err != nil { + return Relationship{}, err + } + r := Relationship{ + ID: id, IssueKey: row.S("issue_key"), Status: row.S("status"), + Parent: Endpoint{row.S("parent_task_id"), row.S("parent_host_id"), row.Opt("parent_cwd")}, + Child: Endpoint{row.S("child_task_id"), row.S("child_host_id"), row.Opt("child_cwd")}, + Generation: row.I("execution_generation"), ScopeRef: row.Opt("scope_ref"), + SupersededBy: row.Opt("superseded_by"), Generations: generations, + } + if err := json.Unmarshal([]byte(row.S("artifact_roots")), &r.ArtifactRoots); err != nil { + return Relationship{}, err + } + if err := json.Unmarshal([]byte(row.S("allowed_recipients")), &r.AllowedRecipients); err != nil { + return Relationship{}, err + } + if r.generation(r.Generation) == nil { + return Relationship{}, refuse(UnknownGeneration, "%s points at generation %d which is not retained", store.PyRepr(id), r.Generation) + } + return r, nil +} + +// RequireActive is registry.require_active. +func RequireActive(ctx context.Context, s *store.Store, id string) (Relationship, error) { + r, err := LoadRelationship(ctx, s, id) + if err != nil { + return r, err + } + if r.Status != "active" { + return r, refuse(RelationshipNotActive, "relationship %s is %s and is never auto-resumed", store.PyRepr(id), store.PyRepr(r.Status)) + } + return r, nil +} + +// ProjectKey is registry.project_key: grouping, never authorization. +func ProjectKey(r Relationship) string { + if cwd, ok := r.Parent.Cwd.(string); ok && cwd != "" { + return pyNormpath(cwd) + } + return "host:" + r.Parent.HostID +} diff --git a/internal/relay/delivery/render.go b/internal/relay/delivery/render.go new file mode 100644 index 00000000..1d66a6e0 --- /dev/null +++ b/internal/relay/delivery/render.go @@ -0,0 +1,367 @@ +package delivery + +import ( + "fmt" + "strings" +) + +// Message kinds (delivery.py). +const ( + Completion = "completion_event" + Revision = "revision_request" + MergeTurnGrant = "merge_turn_grant" + manifestLines = 10 + notRecorded = "not recorded" + noNote = "no note recorded; ask the parent" +) + +var headings = []string{"VIOLATED CRITERION", "WHAT CHANGED", "FIX SCOPE", "PRESERVE", "REVERIFY AND RETURN", "TASK", "SCOPE", "MUST DO", "MUST NOT", "PROOF", "RETURN FORMAT", "DECISION BOUNDARY", "VERDICT"} + +func pyStr(v any) string { + switch t := v.(type) { + case nil: + return "None" + case string: + return t + case bool: + if t { + return "True" + } + return "False" + case int64: + return fmt.Sprint(t) + case float64: + return pyFloat(t) + } + return pyReprValue(v) +} + +// splitlines is str.splitlines. +func splitlines(text string) []string { + var parts []string + var cur strings.Builder + runes := []rune(text) + for i := 0; i < len(runes); i++ { + r := runes[i] + switch r { + case '\n', '\r', '\v', '\f', 0x1c, 0x1d, 0x1e, 0x85, 0x2028, 0x2029: + parts = append(parts, cur.String()) + cur.Reset() + if r == '\r' && i+1 < len(runes) && runes[i+1] == '\n' { + i++ + } + default: + cur.WriteRune(r) + } + } + if cur.Len() > 0 { + parts = append(parts, cur.String()) + } + return parts +} + +// inline is report.inline: a record's text kept on the one line it is spliced into. +func inline(v any) string { + text := pyStr(v) + parts := splitlines(text) + if len(parts) <= 1 && (len(parts) == 0 || parts[0] == text) { + return text + } + var kept []string + for _, p := range parts { + if strings.TrimSpace(p) != "" { + kept = append(kept, p) + } + } + return strings.Join(kept, " / ") +} + +func known(v any) string { + if v == nil || v == "" { + return notRecorded + } + return inline(v) +} + +func unheaded(v any) string { + text := pyStr(v) + probe := strings.TrimSpace(text) + probe = strings.TrimLeft(probe, "-*#>") + probe = strings.Trim(strings.TrimSpace(probe), "*`_") + probe = strings.ToUpper(probe) + for _, name := range headings { + if probe == name || strings.HasPrefix(probe, name+":") { + return `"` + text + `"` + } + } + return text +} + +func notRecordedBecause(what string) string { return notRecorded + ": " + what + "; ask the parent" } + +func findingsWord(count int, noun string) string { + if count == 1 { + return fmt.Sprintf("%d %s", count, noun) + } + return fmt.Sprintf("%d %ss", count, noun) +} + +func series(words []string) string { + if len(words) == 1 { + return words[0] + } + return strings.Join(words[:len(words)-1], ", ") + " and " + words[len(words)-1] +} + +func correctionFindings(receipt Obj) []Obj { + var out []Obj + list, _ := get(receipt, "criteria") + items, _ := list.([]any) + for _, item := range items { + if o, ok := item.(Obj); ok && truthy(func() any { v, _ := get(o, "id"); return v }()) { + out = append(out, o) + } + } + return out +} + +type verdictCounts struct{ fix, owed, met, undecided int } + +func countFindings(findings []Obj) verdictCounts { + var c verdictCounts + for _, f := range findings { + switch v, _ := get(f, "verdict"); v { + case "needs_changes": + c.fix++ + case "unverified": + c.owed++ + case "verified": + c.met++ + default: + c.undecided++ + } + } + return c +} + +func violatedHeading(receipt Obj) string { + findings := correctionFindings(receipt) + if len(findings) == 0 { + return "VIOLATED CRITERION: " + notRecordedBecause("the verdict named no criterion") + } + c := countFindings(findings) + var parts []string + for _, p := range []struct { + n int + text string + }{{c.fix, "marked needs_changes"}, {c.owed, "marked unverified"}, {c.met, "marked verified"}, {c.undecided, "without a disposition"}} { + if p.n > 0 { + parts = append(parts, fmt.Sprintf("%d %s", p.n, p.text)) + } + } + return "VIOLATED CRITERION: of the " + findingsWord(len(findings), "recorded finding") + ", " + strings.Join(parts, ", ") +} + +func whatChangedLines(receipt Obj) []string { + g := func(k string) any { v, _ := get(receipt, k); return v } + return []string{"", + fmt.Sprintf("WHAT CHANGED: submission %s ruled %s and superseded, generation %s opened", known(g("supersedesEvent")), known(g("verdict")), known(g("executionGeneration"))), + fmt.Sprintf(" superseded revision %s, ruled in turn %s", known(g("supersedesRevisionHash")), known(g("verdictTurnId")))} +} + +func fixScopeLines(receipt Obj) []string { + findings := correctionFindings(receipt) + if len(findings) == 0 { + return []string{"", "FIX SCOPE: " + notRecordedBecause("no criterion was named, so nothing bounds a change")} + } + c := countFindings(findings) + unruled := "FIX SCOPE: no finding is marked needs_changes, so nothing is ruled violated" + var heading string + switch { + case c.fix > 0: + heading = fmt.Sprintf("FIX SCOPE: only the %s marked needs_changes, shown or not; %s findings are out of scope", findingsWord(c.fix, "finding"), series([]string{"verified", "unverified"})) + case c.owed > 0: + heading = unruled + "; change nothing but the evidence REVERIFY AND RETURN asks for" + case c.undecided > 0: + heading = unruled + "; change nothing until the parent settles the findings below" + default: + heading = "FIX SCOPE: " + notRecordedBecause("what to change, since every finding is marked verified") + } + lines := []string{"", heading} + if c.undecided > 0 { + pronoun := "them" + if c.undecided == 1 { + pronoun = "it" + } + lines = append(lines, fmt.Sprintf(" %s without a disposition: whether to change %s is not recorded; ask the parent first", findingsWord(c.undecided, "finding"), pronoun)) + } + return append(lines, " anything wider, anything that would discard preserved work, or anything needing authority you were not given comes back to the parent") +} + +func reverifyLines(receipt Obj) []string { + findings := correctionFindings(receipt) + var clauses []string + if len(findings) == 0 { + clauses = []string{"what to re-check is " + notRecordedBecause("the verdict named no criterion")} + } else { + c := countFindings(findings) + if c.fix > 0 { + clauses = append(clauses, "re-check each finding FIX SCOPE names") + } + if c.owed > 0 { + clauses = append(clauses, "show evidence for the "+findingsWord(c.owed, "finding")+" marked unverified") + } + if c.undecided > 0 { + clauses = append(clauses, "settle with the parent whether to change the "+findingsWord(c.undecided, "finding")+" without a disposition") + } + if len(clauses) == 0 { + clauses = append(clauses, "what to re-check is "+notRecordedBecause("every finding is marked verified")) + } + } + return []string{"", "REVERIFY AND RETURN: " + strings.Join(append(clauses, "then hand back as below"), "; ")} +} + +func returnLines(rid string, generation any) []string { + shown := "" + if generation != nil && generation != "" { + shown = inline(generation) + } + return []string{"", + "There is nothing to acknowledge. Contract v1 defines no acknowledgement for this", + "direction and the relay refuses one by kind, so there is no proof to compute and", + "no acknowledgement to send.", + "Answer with your next completion receipt under the new generation:", + fmt.Sprintf(" emit --relationship %s --generation %s --attempt ", rid, shown), + " --outcome ready_for_review --turn-thread --turn-id ", + " --artifact [--continues-anchor ]"} +} + +func restorationLabel(f Obj) string { + if v, _ := get(f, "restoration"); truthy(v) { + return " [restoration block]" + } + return "" +} + +func overflowLine(items []any, eventID string, nameBlock bool) string { + if len(items) <= manifestLines { + return "" + } + hidden := items[manifestLines:] + detail := "" + if nameBlock { + for _, h := range hidden { + if o, ok := h.(Obj); ok { + if v, _ := get(o, "restoration"); truthy(v) { + id, _ := get(o, "id") + detail = ", including the restoration block on " + pyStr(id) + break + } + } + } + } + return fmt.Sprintf(" ... %d more%s; see 'codex-session-relay show --event %s'", len(hidden), detail, eventID) +} + +func renderCompletion(row Row, record Obj, request string) string { + g := func(k string) any { v, _ := get(record, k); return v } + event := row.S("event_id") + lines := []string{ + "[codex-session-relay] verification request", + "requestId: " + request, + "eventId: " + event, + "relationshipId: " + row.S("relationship_id"), + "executionGeneration: " + pyStr(g("executionGeneration")), + "attempt: " + pyStr(g("attempt")), + "outcome: " + pyStr(g("outcome")), + "revisionHash: " + pyStr(g("revisionHash")), + } + if manifest, _ := g("manifest").([]any); len(manifest) > 0 { + lines = append(lines, fmt.Sprintf("deliverables: %d", len(manifest))) + for _, entry := range manifest[:min(len(manifest), manifestLines)] { + o, _ := entry.(Obj) + line := " " + pyStr(func() any { v, _ := get(o, "path"); return v }()) + " sha256=" + pyStr(func() any { v, _ := get(o, "sha256"); return v }()) + if size, ok := get(o, "bytes"); ok && size != nil { + line += " bytes=" + pyStr(size) + } + lines = append(lines, line) + } + if overflow := overflowLine(manifest, event, false); overflow != "" { + lines = append(lines, overflow) + } + } else { + lines = append(lines, "deliverables: none (execution-only outcome)") + } + if ref := g("manifestRef"); truthy(ref) { + lines = append(lines, "manifestRef: "+pyStr(ref)) + } + if criteria, _ := g("criteria").([]any); len(criteria) > 0 { + lines = append(lines, "criteria claimed by the child:") + for _, item := range criteria[:min(len(criteria), manifestLines)] { + o, _ := item.(Obj) + id, _ := get(o, "id") + verdict, _ := get(o, "verdict") + lines = append(lines, " "+pyStr(id)+": "+pyStr(verdict)) + } + if overflow := overflowLine(criteria, event, false); overflow != "" { + lines = append(lines, overflow) + } + } + lines = append(lines, + "", + "To respond, from inside your own turn:", + " claim --event "+event+" --turn ", + " ack-proof --event "+event+" --turn ", + " ack --event "+event+" --ack-turn --ack-proof ", + " verdict --event "+event+" --verdict --verdict-turn ", + "", + "The proof is sha256(eventId|). This message does not and cannot", + "contain that turn id, which is what distinguishes acknowledging from echoing.", + "Full record: codex-session-relay show --event "+event, + ) + return strings.Join(lines, "\n") +} + +func renderRevision(row Row, record Obj, request string) string { + g := func(k string) any { v, _ := get(record, k); return v } + event := row.S("event_id") + lines := []string{ + "[codex-session-relay] revision request", + "requestId: " + request, + "eventId: " + event, + "relationshipId: " + row.S("relationship_id"), + "executionGeneration: " + known(g("executionGeneration")) + " (new)", + "supersedesEvent: " + known(g("supersedesEvent")), + "supersedesRevisionHash: " + known(g("supersedesRevisionHash")), + "verdict: " + known(g("verdict")), + } + findings, _ := g("criteria").([]any) + lines = append(lines, "", violatedHeading(record)) + for _, item := range findings[:min(len(findings), manifestLines)] { + o, _ := item.(Obj) + id, _ := get(o, "id") + if !truthy(id) { + id = "(no id recorded)" + } + verdict, _ := get(o, "verdict") + if !truthy(verdict) { + verdict = "no disposition recorded" + } + note, _ := get(o, "note") + tail := " — " + noNote + if truthy(note) { + tail = " — " + inline(note) + } + lines = append(lines, " "+unheaded(inline(id))+restorationLabel(o)+": "+inline(verdict)+tail) + } + if overflow := overflowLine(findings, event, true); overflow != "" { + lines = append(lines, inline(overflow)) + } + lines = append(lines, whatChangedLines(record)...) + lines = append(lines, fixScopeLines(record)...) + lines = append(lines, "", "PRESERVE: everything FIX SCOPE does not name, verified findings included,", " work this request does not mention and any other task in-flight beside it") + lines = append(lines, reverifyLines(record)...) + lines = append(lines, returnLines(row.S("relationship_id"), g("executionGeneration"))...) + lines = append(lines, "", "Full record: codex-session-relay show --event "+event) + return strings.Join(lines, "\n") +} diff --git a/internal/relay/delivery/rereview_test.go b/internal/relay/delivery/rereview_test.go new file mode 100644 index 00000000..7337e50b --- /dev/null +++ b/internal/relay/delivery/rereview_test.go @@ -0,0 +1,487 @@ +package delivery + +import ( + "context" + "database/sql" + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" +) + +// test_rereview_deadlock.py RRD-1..RRD-9, mirrored test for test (hostloss_harness_test.go mirror +// + testdata/capture.py). The re-review itself (claim_verification's re-claim, record_verdict's +// decided-not-replayed path, the verdict_superseded journal) is delivery's ack.go. The assignment +// view the tests read state through is todo 25's (registry.AssignmentView on codex/crw-154-registry); +// rrState below reads the same rows by assignment.py's _resolve for the states these tests reach, +// and rrMark writes AssignmentView.mark's rows (the journal row is compared with Python's). + +const rrd = "test_rereview_deadlock" + +func rrCriteria(titles ...string) []any { + out := []any{} + for i, title := range titles { + out = append(out, Obj{{Key: "id", Value: []string{"c1", "c2"}[i]}, {Key: "title", Value: title}}) + } + return out +} + +var ( + rrSet = rrCriteria("the endpoint returns the agreed shape", "a malformed request is refused") + rrEdited = rrCriteria("the endpoint returns a COMPLETELY different shape", "a malformed request is refused") + rrPassing = []any{Obj{{Key: "id", Value: "c1"}, {Key: "verdict", Value: "verified"}}, Obj{{Key: "id", Value: "c2"}, {Key: "verdict", Value: "verified"}}} +) + +func rrFinding(verdict, note string) []any { + return []any{Obj{{Key: "id", Value: "c1"}, {Key: "verdict", Value: verdict}, {Key: "note", Value: note}}} +} + +func (h *hl) registerCriteria(set []any) { + h.t.Helper() + _, err := h.ack.Criteria.Register(h.ctx, h.rid, set, "https://linear.app/doc/1") + mustDo(h.t, err) +} + +func (h *hl) setDigest() any { + got, err := h.ack.Criteria.Get(h.ctx, h.rid) + mustDo(h.t, err) + return field(got, "setDigest") +} + +func (h *hl) claim(event, turn string) string { + h.t.Helper() + out, err := h.ack.ClaimVerification(h.ctx, event, turn) + mustDo(h.t, err) + return out +} + +func (h *hl) rule(event, verdict, turn string, findings []any, reason, expected any) (Obj, error) { + return h.ack.RecordVerdict(h.ctx, event, verdict, turn, nil, findings, reason, expected) +} + +func (h *hl) mustRule(event, verdict, turn string, findings []any, reason, expected any) Obj { + h.t.Helper() + out, err := h.rule(event, verdict, turn, findings, reason, expected) + mustDo(h.t, err) + return out +} + +// claimed is ReReviewTestCase.claimed. +func (h *hl) claimed(register bool) string { + event := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + if register { + h.registerCriteria(rrSet) + } + h.attemptOn(event, h.host, nil) + h.clock.Advance(5) + turn := h.host.startTurn(parent, "ack-turn", "inProgress", "") + _, err := h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host) + mustDo(h.t, err) + h.claim(event, turn.TurnID) + return event +} + +func (h *hl) managedVerified() string { + event := h.claimed(true) + h.mustRule(event, "verified", "v1", rrPassing, nil, nil) + return event +} + +func (h *hl) editCriteria() { h.registerCriteria(rrEdited) } + +// reReview is ReReviewTestCase.re_review. +func (h *hl) reReview(event, verdict string, findings []any) Obj { + h.claim(event, "re-review-turn") + if findings == nil { + findings = rrPassing + } + return h.mustRule(event, verdict, "v2", findings, nil, h.setDigest()) +} + +// rrState is AssignmentView.state's state, nextExpectedAction, head, mark and criteria. +func (h *hl) rrState() Obj { + h.t.Helper() + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(h.t, err) + head, err := HeadRevision(h.ctx, h.store, h.rid, r.Generation) + mustDo(h.t, err) + headID := field(head, "eventId") + var verdict Row + if headID != nil { + verdict = h.one("SELECT v.verdict, c.set_digest FROM verdicts v LEFT JOIN verdict_context c ON c.event_id = v.event_id WHERE v.event_id = ?", headID) + } + digest := h.setDigest() + current := verdict == nil || verdict.Opt("set_digest") == digest + var mark Obj + if current && headID != nil && verdict != nil && verdict.S("verdict") == "verified" { + if m := h.one("SELECT * FROM assignment_marks WHERE relationship_id = ? AND event_id = ? AND execution_generation = ? AND revision_hash = ? ORDER BY marked_at LIMIT 1", h.rid, headID, r.Generation, field(head, "revisionHash")); m != nil { + mark = Obj{{Key: "mark", Value: m.S("mark")}, {Key: "eventId", Value: m.S("event_id")}} + } + } + previous := h.one("SELECT v.event_id FROM verdicts v JOIN events e ON e.event_id = v.event_id WHERE e.relationship_id = ? AND e.execution_generation < ? AND v.verdict = 'needs_changes' ORDER BY e.execution_generation DESC LIMIT 1", h.rid, r.Generation) + state := "requested" + switch { + case r.Status == "paused": + state = "paused" + case slices.Contains(ambiguousEvidence, str(head, "evidence")): + state = "ambiguous" + case verdict != nil && verdict.S("verdict") == "verified" && !current: + state = "re_review_needed" + case mark != nil: + state = str(mark, "mark") + case verdict != nil && verdict.S("verdict") == "verified": + state = "verified" + case headID != nil && previous != nil: + state = "corrected" + case headID != nil && h.one("SELECT 1 FROM verification_claims WHERE event_id = ?", headID) != nil: + state = "verifying" + case headID != nil: + state = "received" + case previous != nil: + state = "needs_changes" + } + action := map[string]string{"re_review_needed": "parent_verifies", "verified": "coordinator_integrates", "merged": "none", "paused": "owner_resumes"}[state] + if action == "" { + action = h.nextAction() + } + var reviewed any + if verdict != nil { + reviewed = verdict.Opt("set_digest") + } + var markValue any + if mark != nil { + markValue = mark + } + return Obj{{Key: "state", Value: state}, {Key: "nextExpectedAction", Value: action}, {Key: "head", Value: Obj{{Key: "eventId", Value: headID}}}, {Key: "mark", Value: markValue}, + {Key: "criteria", Value: Obj{{Key: "setDigest", Value: digest}, {Key: "reviewedSetDigest", Value: reviewed}, {Key: "current", Value: current}}}} +} + +// rrMark is AssignmentView.mark(rid, "merged", ...) for a verified, current head: its rows. +func (h *hl) rrMark(event, evidence, actor string) { + h.t.Helper() + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(h.t, err) + head, err := HeadRevision(h.ctx, h.store, h.rid, r.Generation) + mustDo(h.t, err) + if field(head, "eventId") != event { + h.t.Fatalf("the mark names %s but the head is %v", event, field(head, "eventId")) + } + now := h.clock.ISO() + mustDo(h.t, h.store.Transaction(h.ctx, func(ctx context.Context, _ *sql.Conn) error { + if _, err := execSQL(ctx, h.store, "INSERT INTO assignment_marks (relationship_id, mark, event_id, execution_generation, revision_hash, evidence, actor, marked_at) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT(relationship_id, mark, event_id) DO UPDATE SET evidence = excluded.evidence, actor = excluded.actor, marked_at = excluded.marked_at", + h.rid, "merged", event, r.Generation, field(head, "revisionHash"), evidence, actor, now); err != nil { + return err + } + return journal(ctx, h.store, "assignment_marked", h.rid, Obj{{Key: "mark", Value: "merged"}, {Key: "eventId", Value: event}, {Key: "generation", Value: r.Generation}}, now) + })) +} + +// reviewTables are the review-side tables the re-review paths write, beyond mirror's delivery tables. +var reviewTables = []string{"verification_claims", "claim_context", "verdict_context", "canonical_criteria", "verification_mode", "assignment_marks", "relationships"} + +// rrMirror is mirror plus a row-for-row comparison of reviewTables with Python's. +func rrMirror(t *testing.T, name string, body func(h *hl)) { + t.Helper() + mirror(t, rrd, name, func(h *hl) { + body(h) + raw, err := os.ReadFile(filepath.Join(pythonCaptures(t, rrd), name, "capture.json")) + mustDo(t, err) + var python pyCapture + mustDo(t, json.Unmarshal(raw, &python)) + got := normalizeJSON(t, h.tables()).(map[string]any) + want := normalizeJSON(t, python.Tables).(map[string]any) + for _, n := range reviewTables { + requireSameJSON(t, "table "+n, got[n], want[n]) + } + }) +} + +func replayed(record Obj) bool { return truthy(field(record, "_replay")) } + +func Test21_RRD01_a_criteria_edit_after_a_verified_verdict_needs_a_re_review(t *testing.T) { + t.Run("managed verified then edit", func(t *testing.T) { + rrMirror(t, "ReReviewIsReachable.test_the_starting_point_is_the_state_the_view_already_reports", func(h *hl) { + h.managedVerified() + h.eq(str(h.rrState(), "state")) + h.editCriteria() + record := h.rrState() + h.eq(str(record, "state")) + h.eq(str(record, "nextExpectedAction")) + }) + }) + t.Run("legacy verdict then first registration", func(t *testing.T) { + rrMirror(t, "ReviewClaimedBeforeTheEdit.test_registering_criteria_after_a_legacy_verdict_opens_a_re_review", func(h *hl) { + event := h.claimed(false) + h.mustRule(event, "verified", "v1", nil, nil, nil) + h.eq(str(h.rrState(), "state")) + h.registerCriteria(rrSet) + h.eq(str(h.rrState(), "state")) + h.eq(h.claim(event, "re-review-turn")) + h.refusal(h.rule(event, "verified", "v2", rrPassing, nil, nil)) + record := h.mustRule(event, "verified", "v2", rrPassing, nil, h.setDigest()) + h.eq(replayed(record)) + h.eq(str(h.rrState(), "state")) + }) + }) +} + +func (h *hl) claimHolder(event string) string { + return h.one("SELECT * FROM verification_claims WHERE event_id = ?", event).S("claim_turn_id") +} + +func (h *hl) boundDigest(event string) any { + got, err := h.ack.Criteria.BoundDigest(h.ctx, event) + mustDo(h.t, err) + return got +} + +func Test21_RRD02_a_re_claim_rebinds_once_and_only_once(t *testing.T) { + t.Run("first re-claim", func(t *testing.T) { + rrMirror(t, "ReReviewIsReachable.test_the_review_can_be_claimed_again_onto_the_current_set", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + h.eq(h.boundDigest(event)) + h.eq(h.claimHolder(event)) + }) + }) + t.Run("second re-claim", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_re_claiming_does_not_become_an_unlimited_claim", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + h.eq(h.claim(event, "third-reviewer-turn")) + h.eq(h.claimHolder(event)) + }) + }) + t.Run("after a landed re-review", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_a_landed_re_review_does_not_leave_the_claim_open", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.reReview(event, "verified", nil) + h.eq(h.claim(event, "fourth-turn")) + }) + }) + t.Run("after an attested re-review", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_an_attested_re_review_does_not_leave_the_claim_open_either", func(h *hl) { + event := h.claimed(false) + h.mustRule(event, "verified", "v1", nil, nil, nil) + h.registerCriteria(rrSet) + record := h.mustRule(event, "verified", "v2", rrPassing, nil, h.setDigest()) + h.eq(replayed(record)) + h.eq(str(h.rrState(), "state")) + h.eq(h.boundDigest(event)) + h.eq(h.claim(event, "fifth-turn")) + }) + }) + t.Run("unchanged set", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_an_unchanged_criteria_set_still_refuses_a_second_claim", func(h *hl) { + event := h.managedVerified() + h.eq(h.claim(event, "another-turn")) + h.eq(h.count("SELECT COUNT(*) AS c FROM verification_claims")) + }) + }) +} + +func Test21_RRD03_a_re_review_is_decided_not_replayed(t *testing.T) { + rrMirror(t, "ReReviewIsReachable.test_a_re_review_is_decided_rather_than_replayed", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + record := h.reReview(event, "verified", nil) + h.eq(replayed(record)) + h.eq(field(record, "verdictTurnId")) + }) + rrMirror(t, "ReReviewIsReachable.test_a_re_review_returns_the_assignment_to_verified", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.reReview(event, "verified", nil) + record := h.rrState() + h.eq(str(record, "state")) + h.eq(field(sub(record, "criteria"), "current")) + h.eq(field(sub(record, "criteria"), "reviewedSetDigest")) + }) + rrMirror(t, "ReReviewIsReachable.test_the_child_never_has_to_touch_an_unrelated_byte", func(h *hl) { + event := h.managedVerified() + before := h.count("SELECT COUNT(*) AS c FROM events") + h.editCriteria() + record := h.reReview(event, "verified", nil) + h.eq(replayed(record)) + h.eq(h.count("SELECT COUNT(*) AS c FROM events")) + h.eq(field(sub(h.rrState(), "head"), "eventId")) + _ = before + }) + rrMirror(t, "ReReviewIsReachable.test_a_re_review_can_also_ask_for_changes", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + record := h.reReview(event, "needs_changes", rrFinding("needs_changes", "the new shape is missing")) + h.eq(field(record, "nextExecutionGeneration")) + r, err := LoadRelationship(h.ctx, h.store, h.rid) + mustDo(t, err) + h.eq(r.Generation) + }) +} + +func Test21_RRD04_the_superseded_verdict_stays_on_the_record(t *testing.T) { + rrMirror(t, "ReReviewIsReachable.test_the_superseded_verdict_stays_on_the_record", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.reReview(event, "verified", nil) + entry := h.one("SELECT detail FROM journal WHERE kind = 'verdict_superseded' AND subject = ?", event) + h.eq(entry != nil) + superseded := sub(loadsObj(entry.S("detail")), "supersededVerdict") + h.eq(field(superseded, "verdictTurnId")) + h.eq(field(superseded, "verdict")) + }) +} + +func Test21_RRD05_an_earlier_merge_mark_is_state_again_once_the_re_review_lands(t *testing.T) { + rrMirror(t, "ReReviewIsReachable.test_an_earlier_merge_mark_is_state_again_once_the_re_review_lands", func(h *hl) { + event := h.managedVerified() + h.rrMark(event, "merged as abc1234", parent) + h.eq(str(h.rrState(), "state")) + h.editCriteria() + h.eq(str(h.rrState(), "state")) + h.reReview(event, "verified", nil) + record := h.rrState() + h.eq(str(record, "state")) + h.eq(field(sub(record, "mark"), "eventId")) + }) +} + +func Test21_RRD06_a_review_claimed_before_the_edit_finishes_once_claimed_again(t *testing.T) { + rrMirror(t, "ReviewClaimedBeforeTheEdit.test_the_ruling_is_still_refused_until_the_review_is_claimed_again", func(h *hl) { + event := h.claimed(true) + h.editCriteria() + h.refusal(h.rule(event, "verified", "v1", rrPassing, nil, nil)) + }) + rrMirror(t, "ReviewClaimedBeforeTheEdit.test_claiming_it_again_lets_the_review_finish", func(h *hl) { + event := h.claimed(true) + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + record := h.mustRule(event, "verified", "v1", rrPassing, nil, nil) + h.eq(replayed(record)) + h.eq(str(h.rrState(), "state")) + }) + rrMirror(t, "ReviewClaimedBeforeTheEdit.test_an_unruled_re_review_needs_no_stated_digest", func(h *hl) { + event := h.claimed(true) + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + record := h.mustRule(event, "verified", "v1", rrPassing, nil, nil) + h.eq(field(record, "verdict")) + h.eq(replayed(record)) + }) +} + +func Test21_RRD07_a_recorded_verdict_or_old_findings_cannot_certify_the_new_set(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_a_recorded_verdict_cannot_be_re_submitted_as_its_own_re_review", func(h *hl) { + event := h.managedVerified() + old := h.setDigest() + h.editCriteria() + h.claim(event, "re-review-turn") + h.refusal(h.rule(event, "verified", "v1", rrPassing, nil, nil)) + h.refusal(h.rule(event, "verified", "v1", rrPassing, nil, old)) + h.eq(str(h.rrState(), "state")) + }) + rrMirror(t, "ProtectionsThatMustSurvive.test_findings_made_against_the_old_wording_are_still_refused", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.refusal(h.rule(event, "verified", "v2", rrPassing, nil, nil)) + }) +} + +func Test21_RRD08_a_re_review_cannot_replace_a_certification_with_no_state(t *testing.T) { + t.Run("unverified", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_a_re_review_cannot_strand_the_assignment_as_unverified", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.claim(event, "re-review-turn") + h.refusal(h.rule(event, "unverified", "v2", rrFinding("unverified", "could not reach it"), nil, h.setDigest())) + h.eq(str(h.rrState(), "state")) + h.eq(str(h.rrState(), "nextExpectedAction")) + }) + }) + t.Run("aborted", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_a_re_review_cannot_strand_the_assignment_as_aborted", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.claim(event, "re-review-turn") + h.refusal(h.rule(event, "aborted", "v2", nil, "stopping", h.setDigest())) + h.eq(str(h.rrState(), "state")) + }) + }) + t.Run("first ruling unverified", func(t *testing.T) { + rrMirror(t, "AReClaimIsNotAFreePass.test_an_event_with_no_ruling_can_still_be_recorded_unverified", func(h *hl) { + event := h.claimed(true) + record := h.mustRule(event, "unverified", "v1", rrFinding("unverified", "could not reach it"), nil, nil) + h.eq(field(record, "verdict")) + }) + }) +} + +func Test21_RRD09_the_protections_that_must_survive(t *testing.T) { + t.Run("unchanged set replays", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_an_unchanged_criteria_set_still_replays", func(h *hl) { + event := h.managedVerified() + again := h.mustRule(event, "needs_changes", "v2", rrFinding("needs_changes", "on reflection, no"), nil, nil) + h.eq(replayed(again)) + h.eq(field(again, "verdictTurnId")) + h.eq(field(again, "verdict")) + }) + }) + t.Run("second verified on an unchanged set", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_a_second_verified_ruling_on_an_unchanged_set_still_replays", func(h *hl) { + event := h.managedVerified() + again := h.mustRule(event, "verified", "v2", rrPassing, nil, nil) + h.eq(replayed(again)) + h.eq(field(again, "verdictTurnId")) + }) + }) + t.Run("generation advanced", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_a_criteria_edit_does_not_reopen_a_verdict_the_generation_left_behind", func(h *hl) { + event := h.managedVerified() + h.openGeneration("later", "needs_changes_revision", "later-turn") + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + again := h.mustRule(event, "unverified", "v2", rrFinding("unverified", "could not reach it"), nil, nil) + h.eq(replayed(again)) + h.eq(field(again, "verdict")) + }) + }) + t.Run("prior unverified", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_a_criteria_edit_after_unverified_does_not_reopen_the_old_event", func(h *hl) { + event := h.claimed(true) + h.mustRule(event, "unverified", "v1", rrFinding("unverified", "no access"), nil, nil) + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + again := h.mustRule(event, "verified", "v2", rrPassing, nil, h.setDigest()) + h.eq(replayed(again)) + h.eq(field(again, "verdict")) + }) + }) + t.Run("prior needs_changes", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_a_criteria_edit_after_needs_changes_does_not_reopen_the_old_event", func(h *hl) { + event := h.queuedEvent(regOpts{recipients: []string{parent, child}}) + h.registerCriteria(rrSet) + h.attemptOn(event, h.host, nil) + h.clock.Advance(5) + turn := h.host.startTurn(parent, "ack-turn", "inProgress", "") + _, err := h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host) + mustDo(t, err) + h.claim(event, turn.TurnID) + h.mustRule(event, "needs_changes", "v1", rrFinding("needs_changes", "fix it"), nil, nil) + h.editCriteria() + h.eq(h.claim(event, "re-review-turn")) + again := h.mustRule(event, "verified", "v2", rrPassing, nil, nil) + h.eq(replayed(again)) + }) + }) + t.Run("paused", func(t *testing.T) { + rrMirror(t, "ProtectionsThatMustSurvive.test_a_paused_assignment_is_not_re_reviewable", func(h *hl) { + event := h.managedVerified() + h.editCriteria() + h.setStatusBy("paused", parent) + h.eq(h.claim(event, "re-review-turn")) + }) + }) +} diff --git a/internal/relay/delivery/revision_test.go b/internal/relay/delivery/revision_test.go new file mode 100644 index 00000000..2a39d486 --- /dev/null +++ b/internal/relay/delivery/revision_test.go @@ -0,0 +1,211 @@ +package delivery + +import ( + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_revision_roundtrip.py RVR-1..RVR-5. + +func (v *vcu) requestCorrection(event string) Obj { + _, err := v.ack.RecordVerdict(v.ctx, event, "needs_changes", "review-"+event, nil, []any{finding("c1", "needs_changes", "fix the output")}, nil, nil) + mustDo(v.t, err) + r, err := LoadRelationship(v.ctx, v.store, v.rid) + mustDo(v.t, err) + req := v.one("SELECT * FROM events WHERE relationship_id = ? AND execution_generation = ? AND outcome = 'revision_request'", v.rid, r.Generation) + v.mustAttempt(req.S("event_id"), nil) + bound, err := v.ack.BindDispatchedRevision(v.ctx, req.S("event_id")) + mustDo(v.t, err) + if str(bound, "anchorState") != "bound" || v.host.sends[len(v.host.sends)-1].thread != child { + v.t.Fatalf("bound %v", bound) + } + v.host.finishTurn(child, str(bound, "dispatchTurnId"), "completed") + return loadsObj(req.S("receipt")) +} + +func (v *vcu) emitCorrection(predecessor any, text string) Obj { + if text == "" { + text = "corrected output" + } + r, err := LoadRelationship(v.ctx, v.store, v.rid) + mustDo(v.t, err) + turn := turnRef{child, r.generation(r.Generation).S("dispatch_turn_id"), "completed"} + payload := v.readyPayload(v.rid, r.Generation, []string{v.artifact("out.txt", text)}, 1, turn) + var supersedes *string + if s, ok := predecessor.(string); ok { + supersedes = &s + } + _, err = v.accept(payload, store.AcceptOptions{SupersedesRevision: supersedes}) + mustDo(v.t, err) + return payload +} + +func (v *vcu) acknowledgeCorrection(payload Obj) { + e := str(payload, "eventId") + _, err := v.delivery.Enqueue(v.ctx, e, "", "") + mustDo(v.t, err) + v.mustAttempt(e, nil) + v.clock.Advance(5) + turn := v.host.startTurn(parent, "", "inProgress", "") + _, err = v.ack.Acknowledge(v.ctx, e, turn.TurnID, AckProof(e, turn.TurnID), true, nil, v.host) + mustDo(v.t, err) + if v.one("SELECT verified FROM acks WHERE event_id = ?", e).S("verified") != "verified" { + v.t.Fatal("the correction's acknowledgement is verified") + } +} + +func (v *vcu) headOf(g int64) Obj { + h, err := HeadRevision(v.ctx, v.store, v.rid, g) + mustDo(v.t, err) + return h +} + +func runRVR(t *testing.T, mode string, goSide func(v *vcu, out map[string]any), args ...string) { + tree := t.TempDir() + python := runPython(t, tree, "rvr", append([]string{mode}, args...)...) + v := newVCU(t, tree) + out := map[string]any{} + goSide(v, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + requireSameTables(t, v.fixture, python) +} + +func TestRVR01_a_correction_by_the_same_child_supersedes_then_verifies_and_replays(t *testing.T) { + verified := []any{finding("c1", "verified", "")} + t.Run("roundtrip and replay", func(t *testing.T) { + runRVR(t, "roundtrip", func(v *vcu, out map[string]any) { + first := v.acknowledged("") + req := v.requestCorrection(first) + corr := v.emitCorrection(str(req, "supersedesRevisionHash"), "") + v.acknowledgeCorrection(corr) + out["final"] = v.verdict(str(corr, "eventId"), "verified", "review-corrected", verified, nil, nil) + out["head"] = v.headOf(2) + out["replay"] = v.verdict(first, "needs_changes", "replayed-review", nil, nil, nil) + r, _ := LoadRelationship(v.ctx, v.store, v.rid) + out["generation"] = r.Generation + if str(out["head"].(Obj), "evidence") != Chain || v.count("SELECT COUNT(*) AS c FROM events WHERE outcome = 'revision_request'") != 1 { + t.Fatal("declared chain, one revision request") + } + }) + }) + t.Run("repeated corrections chain to the immediate request", func(t *testing.T) { + runRVR(t, "repeated", func(v *vcu, out map[string]any) { + first := v.acknowledged("") + req := v.requestCorrection(first) + second := v.emitCorrection(str(req, "supersedesRevisionHash"), "second output") + v.acknowledgeCorrection(second) + req2 := v.requestCorrection(str(second, "eventId")) + third := v.emitCorrection(str(req2, "supersedesRevisionHash"), "third output") + v.acknowledgeCorrection(third) + out["final"] = v.verdict(str(third, "eventId"), "verified", "review-third", verified, nil, nil) + }) + }) + t.Run("a current-generation chain extends the correction", func(t *testing.T) { + runRVR(t, "extend", func(v *vcu, out map[string]any) { + req := v.requestCorrection(v.acknowledged("")) + corr := v.emitCorrection(str(req, "supersedesRevisionHash"), "first fix") + latest := v.emitCorrection(str(corr, "revisionHash"), "refined fix") + v.acknowledgeCorrection(latest) + out["final"] = v.verdict(str(latest, "eventId"), "verified", "final-review", verified, nil, nil) + }) + }) +} + +func TestRVR02_only_the_requested_result_anchors_a_correction(t *testing.T) { + for _, tc := range []struct { + mode string + run func(v *vcu) + }{ + {"historical", func(v *vcu) { first := v.acknowledged(""); v.advance(); v.emitCorrection(v.revisionHash(first), "") }}, + {"unknown", func(v *vcu) { + v.requestCorrection(v.acknowledged("")) + v.emitCorrection("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "") + }}, + {"earlier", func(v *vcu) { + first := v.acknowledged("") + h := v.revisionHash(first) + s := v.emitCorrection(h, "successor") + v.acknowledgeCorrection(s) + v.requestCorrection(str(s, "eventId")) + v.emitCorrection(h, "") + }}, + {"other_rel", func(v *vcu) { + first := v.acknowledged("first relationship output") + h := v.revisionHash(first) + v.requestCorrection(first) + v.rid = v.register(regOpts{issue: "REL-OTHER", dispatchRequest: "other-assignment", recipients: []string{parent, child}}) + initial := v.emitCorrection(nil, "other relationship output") + v.acknowledgeCorrection(initial) + v.requestCorrection(str(initial, "eventId")) + v.emitCorrection(h, "") + }}, + {"suppressed", func(v *vcu) { + req := v.requestCorrection(v.acknowledged("")) + _, err := execSQL(v.ctx, v.store, "UPDATE events SET suppressed_reason = 'withdrawn' WHERE event_id = ?", str(req, "eventId")) + mustDo(v.t, err) + v.emitCorrection(str(req, "supersedesRevisionHash"), "") + }}, + } { + t.Run(tc.mode, func(t *testing.T) { + runRVR(t, tc.mode, func(v *vcu, out map[string]any) { + tc.run(v) + out["head"] = v.headOf(2) + if str(out["head"].(Obj), "evidence") != UnknownPredecessor { + t.Fatalf("head %v", out["head"]) + } + }) + }) + } +} + +func TestRVR03_two_corrections_of_the_requested_result_are_a_fork(t *testing.T) { + runRVR(t, "fork", func(v *vcu, out map[string]any) { + req := v.requestCorrection(v.acknowledged("")) + v.emitCorrection(str(req, "supersedesRevisionHash"), "one") + v.emitCorrection(str(req, "supersedesRevisionHash"), "two") + out["head"] = v.headOf(2) + if id, _ := get(out["head"].(Obj), "eventId"); id != nil || str(out["head"].(Obj), "evidence") != Fork { + t.Fatal("fork with no head") + } + }) +} + +func TestRVR04_an_old_unruled_event_is_not_made_current_by_a_correction(t *testing.T) { + runRVR(t, "old_unruled", func(v *vcu, out map[string]any) { + first := v.acknowledged("") + s := v.emitCorrection(v.revisionHash(first), "successor") + v.acknowledgeCorrection(s) + req := v.requestCorrection(str(s, "eventId")) + v.emitCorrection(str(req, "supersedesRevisionHash"), "") + out["r"] = v.verdict(first, "verified", "late-review", nil, nil, nil) + if out["r"].(map[string]any)["reason"] != StaleGeneration { + t.Fatal("stale_generation") + } + }) +} + +func TestRVR05_the_correction_has_its_own_anchor_in_the_new_generation(t *testing.T) { + for _, reemit := range []bool{false, true} { + name := map[bool]string{false: "queued correction, no completion yet", true: "after the child re-emits"}[reemit] + t.Run(name, func(t *testing.T) { + args := []string{} + if reemit { + args = []string{"reemit"} + } + runRVR(t, "projection", func(v *vcu, out map[string]any) { + first := v.acknowledged("") + req := v.requestCorrection(first) + out["request"] = req + if reemit { + out["corrected"] = v.emitCorrection(str(req, "supersedesRevisionHash"), "") + } + p, err := Projection(v.ctx, v.delivery, v.rid) + mustDo(t, err) + out["projection"] = p + }, args...) + }) + } +} diff --git a/internal/relay/delivery/rows.go b/internal/relay/delivery/rows.go new file mode 100644 index 00000000..a10d59d9 --- /dev/null +++ b/internal/relay/delivery/rows.go @@ -0,0 +1,122 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Row is one SQLite row by column name, as Python's sqlite3.Row is read. +type Row map[string]any + +func (r Row) S(column string) string { + switch v := r[column].(type) { + case string: + return v + case []byte: + return string(v) + } + return "" +} + +// N reports whether column is NULL. +func (r Row) N(column string) bool { return r[column] == nil } + +func (r Row) I(column string) int64 { + switch v := r[column].(type) { + case int64: + return v + case float64: + return int64(v) + } + return 0 +} + +func (r Row) F(column string) float64 { + switch v := r[column].(type) { + case int64: + return float64(v) + case float64: + return v + } + return 0 +} + +// Opt is the column as a JSON value: nil for NULL. +func (r Row) Opt(column string) any { + if b, ok := r[column].([]byte); ok { + return string(b) + } + return r[column] +} + +// all reads every row through the ctx-aware querier and closes the cursor before returning. +func all(ctx context.Context, s *store.Store, query string, args ...any) (_ []Row, err error) { + rows, err := s.Q(ctx).QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { err = errors.Join(err, rows.Close()) }() + columns, err := rows.Columns() + if err != nil { + return nil, err + } + var out []Row + for rows.Next() { + values := make([]any, len(columns)) + pointers := make([]any, len(columns)) + for i := range values { + pointers[i] = &values[i] + } + if err := rows.Scan(pointers...); err != nil { + return nil, err + } + row := Row{} + for i, c := range columns { + if b, ok := values[i].([]byte); ok { + values[i] = string(b) + } + row[c] = values[i] + } + out = append(out, row) + } + return out, rows.Err() +} + +// one reads the first row, or nil. +func one(ctx context.Context, s *store.Store, query string, args ...any) (Row, error) { + rows, err := all(ctx, s, query, args...) + if err != nil || len(rows) == 0 { + return nil, err + } + return rows[0], nil +} + +func execSQL(ctx context.Context, s *store.Store, query string, args ...any) (int64, error) { + result, err := s.Q(ctx).ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +func nullable(v any) any { + if s, ok := v.(string); ok && s == "" { + return nil + } + return v +} + +// journal is store.journal inside the caller's transaction. +func journal(ctx context.Context, s *store.Store, kind, subject string, detail any, at string) error { + text, ok := detail.(string) + if !ok { + text = dumps(detail) + } + _, err := s.Q(ctx).ExecContext(ctx, `INSERT INTO journal (at, kind, subject, detail) VALUES (?,?,?,?)`, at, kind, subject, text) + return err +} + +type sqlConn = sql.Conn diff --git a/internal/relay/delivery/scheduler.go b/internal/relay/delivery/scheduler.go new file mode 100644 index 00000000..b1437413 --- /dev/null +++ b/internal/relay/delivery/scheduler.go @@ -0,0 +1,145 @@ +package delivery + +import ( + "context" + "database/sql" + "slices" + "strconv" +) + +// TickCounts are the delivery counters of the daemon's TickReport. +type TickCounts struct { + Delivered, Deferred, Skipped int + Notes []string +} + +// Scheduler is the delivery pass of RelayDaemon.tick (daemon._deliver), with its persisted +// rotation cursors. The daemon (todo 29) owns the rest of the tick and calls this pass. +type Scheduler struct { + Delivery *Service + Ack *Ack + // MaxSendsTick is policy.max_sends_per_tick: 0 means its default 4, a negative value 0. + MaxSendsTick int +} + +func (sc *Scheduler) cursor(ctx context.Context, listing string, size int) (int, error) { + if size <= 0 { + return 0, nil + } + row, err := one(ctx, sc.Delivery.Store, "SELECT cursor FROM discovery_cursors WHERE task_id = 'scheduler' AND listing = ?", listing) + if err != nil || row == nil || row.N("cursor") { + return 0, err + } + n, convErr := strconv.Atoi(row.S("cursor")) + if convErr != nil { + return 0, nil + } + return n % size, nil +} + +func (sc *Scheduler) advance(ctx context.Context, listing string, by, size int) error { + if size <= 0 { + return nil + } + current, err := sc.cursor(ctx, listing, size) + if err != nil { + return err + } + position := (current + max(1, by)) % size + return sc.Delivery.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := execSQL(ctx, sc.Delivery.Store, "INSERT INTO discovery_cursors (task_id, listing, cursor, updated_at) VALUES ('scheduler',?,?,?) ON CONFLICT(task_id, listing) DO UPDATE SET cursor = excluded.cursor, updated_at = excluded.updated_at", listing, strconv.Itoa(position), sc.Delivery.Clock.ISO()) + return err + }) +} + +// Deliver is daemon._deliver: a fair slice, one struggling parent never spending the budget. +func (sc *Scheduler) Deliver(ctx context.Context, adapter Adapter, now float64, report *TickCounts) error { + d := sc.Delivery + parents, err := d.EligibleParents(ctx, now) + if err != nil || len(parents) == 0 { + return err + } + cursor, err := sc.cursor(ctx, "delivery_parents", len(parents)) + if err != nil { + return err + } + totals := map[string]int{} + offsets := map[string]int{} + for _, p := range parents { + n, err := d.EligibleCount(ctx, p, now) + if err != nil { + return err + } + totals[p] = int(n) + if n > 0 { + if offsets[p], err = sc.cursor(ctx, "deliver:"+p, int(n)); err != nil { + return err + } + } + } + limit := sc.MaxSendsTick + if limit == 0 { + limit = 4 + } + if limit < 0 { + limit = 0 + } + eligible, err := d.Eligible(ctx, now, limit, 0, cursor, offsets) + if err != nil { + return err + } + if err := sc.advance(ctx, "delivery_parents", 1, len(parents)); err != nil { + return err + } + struggling := map[string]bool{} + attempted := map[string]int{} + var order []string + for _, row := range eligible { + p := row.S("parent_task_id") + if struggling[p] { + report.Skipped++ + continue + } + if !slices.Contains(order, p) { + order = append(order, p) + } + attempted[p]++ + record, err := d.Attempt(ctx, row.S("event_id"), adapter, &now, "") + if err != nil { + report.Notes = append(report.Notes, "delivery refused for "+row.S("event_id")+": "+err.Error()) + struggling[p] = true + continue + } + if record == nil { + struggling[p] = true + report.Deferred++ + continue + } + state := str(record, "deliveryState") + if state == HeldUncertain || state == DeferredBusy || state == WithheldPreSend { + struggling[p] = true + } + if v, _ := get(record, "withheldReason"); truthy(v) { + report.Deferred++ + continue + } + if str(record, "sendAttempted") == "no" { + report.Skipped++ + } else { + report.Delivered++ + } + if row.S("kind") == Revision && state == Dispatched && sc.Ack != nil { + if _, err := sc.Ack.BindDispatchedRevision(ctx, row.S("event_id")); err != nil { + report.Notes = append(report.Notes, "anchor binding failed: "+err.Error()) + } + } + } + for _, p := range order { + if totals[p] > 0 { + if err := sc.advance(ctx, "deliver:"+p, attempted[p], totals[p]); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/relay/delivery/scope.go b/internal/relay/delivery/scope.go new file mode 100644 index 00000000..eee15282 --- /dev/null +++ b/internal/relay/delivery/scope.go @@ -0,0 +1,77 @@ +package delivery + +import ( + "path" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +func pyNormpath(p string) string { + cleaned := path.Clean(p) + if strings.HasPrefix(p, "//") && !strings.HasPrefix(p, "///") { + return "/" + cleaned + } + return cleaned +} + +func within(root, candidate string) bool { + root, candidate = pyNormpath(root), pyNormpath(candidate) + if root == "/" { + return strings.HasPrefix(candidate, "/") + } + return candidate == root || strings.HasPrefix(candidate, root+"/") +} + +func reprList(items []string) string { + quoted := make([]string, len(items)) + for i, s := range items { + quoted[i] = store.PyRepr(s) + } + return "[" + strings.Join(quoted, ", ") + "]" +} + +// assertAssignmentDelivery is scope.assert_assignment_delivery: a delivery belongs to ONE +// assignment and goes to that assignment's own endpoint. +func assertAssignmentDelivery(r Relationship, kind, recipient string, recipientThread *string, eventRelationship *string, manifestPaths []string) error { + rid := r.ID + if eventRelationship != nil && *eventRelationship != rid { + return refuse(RecipientNotAuthorized, "event belongs to relationship %s, not %s", store.PyRepr(*eventRelationship), store.PyRepr(rid)) + } + var expected string + switch kind { + case Revision: + expected = r.Child.TaskID + case Completion, MergeTurnGrant: + expected = r.Parent.TaskID + default: + return refuse(RecipientNotAuthorized, "%s is not a delivery direction this contract defines, so there is no authorized recipient for it", store.PyRepr(kind)) + } + if recipient != expected { + direction := "parent" + if kind == Revision { + direction = "child" + } + return refuse(RecipientNotAuthorized, "a %s for %s goes to its own %s %s, not to %s", kind, store.PyRepr(rid), direction, store.PyRepr(expected), store.PyRepr(recipient)) + } + if recipientThread != nil && *recipientThread != recipient { + return refuse(RecipientNotAuthorized, "the native thread %s is not the recipient task %s", store.PyRepr(*recipientThread), store.PyRepr(recipient)) + } + if !slices.Contains(r.AllowedRecipients, recipient) { + return refuse(RecipientNotAuthorized, "recipient %s is not in the relationship's allowed recipients", store.PyRepr(recipient)) + } + for _, p := range manifestPaths { + ok := false + for _, root := range r.ArtifactRoots { + if within(root, p) { + ok = true + break + } + } + if !ok { + return refuse(ScopeEscape, "%s lies outside every authorized root %s", store.PyRepr(p), reprList(r.ArtifactRoots)) + } + } + return nil +} diff --git a/internal/relay/delivery/selection.go b/internal/relay/delivery/selection.go new file mode 100644 index 00000000..c2e17276 --- /dev/null +++ b/internal/relay/delivery/selection.go @@ -0,0 +1,117 @@ +package delivery + +import ( + "os" + "path/filepath" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// selectionRefusal is cli._selection_refusal: what is wrong with the store this run resolved, +// as the JSON refusal every non-exempt command prints before its handler (exit 2), or nil. + +const stateEnv = "CODEX_SESSION_RELAY_STATE" + +func program() string { + name := filepath.Base(os.Args[0]) + if filepath.Dir(os.Args[0]) != "." { + return shellQuote(os.Args[0]) + } + return shellQuote(name) +} + +func withoutStateEnv() string { + if _, ok := os.LookupEnv(stateEnv); ok { + return "env -u " + stateEnv + " " + } + return "" +} + +func selectionRefusal(selection store.StateSelection, socket string) Obj { + if socket != "" { + if _, err := os.Stat(selection.DBPath()); err == nil { + recorded := store.StoreSocket(selection.DBPath()) + wanted, err := store.CanonicalSocket(socket) + if err == nil && recorded != "" && recorded != wanted { + return Obj{{Key: "error", Value: "refused"}, {Key: "reason", Value: "state_directory_serves_another_socket"}, + {Key: "detail", Value: "this store records a different App Server socket; serving the requested one from it would expose one installation's assignments through another"}, + {Key: "recordedSocket", Value: recorded}, {Key: "requestedSocket", Value: wanted}, {Key: "stateDirectory", Value: selection.Path}, + {Key: "recover", Value: wrongSocketRecovery(selection, recorded, wanted)}, + {Key: "note", Value: "using a store does not rewrite the socket it recorded, so neither command here adopts anything; choose the matching pair"}} + } + } + } + if len(selection.Ambiguous) == 0 && len(selection.Unidentified) == 0 { + return nil + } + contested := len(selection.Ambiguous) > 0 + candidates := selection.Unidentified + reason, detail := "unidentified_state_directory", "a store here records no socket, so it cannot be ruled out as this one's; creating a new store beside it would hide it permanently" + if contested { + candidates = selection.Ambiguous + reason, detail = "ambiguous_state_directory", "more than one store already records this socket, and creating a new one here would hide them both" + } + var socketPath any + if socket != "" { + socketPath = socket + } + list := make([]any, len(candidates)) + for i, c := range candidates { + list[i] = c + } + return Obj{{Key: "error", Value: "refused"}, {Key: "reason", Value: reason}, {Key: "detail", Value: detail}, {Key: "socketPath", Value: socketPath}, + {Key: "candidates", Value: list}, {Key: "wouldHaveCreated", Value: selection.DBPath()}, {Key: "recover", Value: recoveryLines(selection, socket, candidates, contested)}} +} + +func recoveryLines(selection store.StateSelection, socket string, candidates []string, contested bool) []any { + prog := program() + socketFlag := "" + if socket != "" { + socketFlag = " --socket=" + shellQuote(socket) + } + lines := []any{prog + socketFlag + " doctor", " lists the candidates under siblingStores"} + for _, c := range candidates { + lines = append(lines, prog+" --state="+shellQuote(c)+socketFlag+" doctor", prog+" --state="+shellQuote(c)+socketFlag+" service status") + } + lines = append(lines, " service status groups by project, so the candidate holding the assignments you expect is the one to keep") + if contested { + return append(lines, " then pass --state= on EVERY participant of this assignment: both stores still record this socket, so default discovery keeps refusing until one of them is retired") + } + return append(lines, " then pass --state="+shellQuote(selection.Path)+" once to create the new store deliberately, or --state= to keep using it") +} + +func wrongSocketRecovery(selection store.StateSelection, recorded, wanted string) []any { + lines := []any{ + program() + " --state=" + shellQuote(selection.Path) + " --socket=" + shellQuote(recorded) + " doctor", + " reads this store under the socket it actually records", + withoutStateEnv() + program() + " --socket=" + shellQuote(wanted) + " doctor", + " discovers by socket alone, ignoring any pinned directory", + } + pinned := os.Getenv(stateEnv) + if pinned == "" { + return lines + } + expanded := pinned + if strings.HasPrefix(pinned, "~") { + home, _ := os.UserHomeDir() + if pinned != "~" && !strings.HasPrefix(pinned, "~/") { + return append(lines, " "+stateEnv+" is set to "+store.PyRepr(pinned)+", which names a home directory that does not resolve on this host, so it is not offered as a candidate") + } + expanded = home + pinned[1:] + } + resolved, err := filepath.Abs(expanded) + if err == nil { + if real, err := filepath.EvalSymlinks(resolved); err == nil { + resolved = real + } + } + current, _ := filepath.Abs(selection.Path) + if real, err := filepath.EvalSymlinks(current); err == nil { + current = real + } + if resolved != current { + lines = append(lines, program()+" --state="+shellQuote(resolved)+" --socket="+shellQuote(wanted)+" doctor", " reads the directory "+stateEnv+" names, which --state overrode on this run") + } + return lines +} diff --git a/internal/relay/delivery/selection_test.go b/internal/relay/delivery/selection_test.go new file mode 100644 index 00000000..5931912e --- /dev/null +++ b/internal/relay/delivery/selection_test.go @@ -0,0 +1,75 @@ +package delivery + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// The store-selection refusals every delivery command prints before its handler (cli.py +// _selection_refusal): ambiguous, unidentified and wrong-socket, byte for byte with Python's +// once the program name each side prints for itself is replaced by one token. +func TestCLI_store_selection_refusals_match_python(t *testing.T) { + root := repoRoot(t) + for _, tc := range []struct { + name string + setup func(t *testing.T, xs string) + state bool + }{ + {"unidentified", func(t *testing.T, xs string) { + dir := filepath.Join(xs, "codex-session-relay", "0123456789abcdef") + mustDo(t, os.MkdirAll(dir, 0o700)) + py := exec.Command("uv", "run", "--no-sync", "python", "-c", "import sys;from codex_session_relay.store import Store;Store(sys.argv[1]).close()", filepath.Join(dir, "relay.sqlite3")) + py.Dir = filepath.Join(root, "packages", "codex-session-relay") + mustDo(t, py.Run()) + }, false}, + {"wrong socket", func(t *testing.T, xs string) {}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + var outs [2]string + var codes [2]int + sockets := t.TempDir() + for i, python := range []bool{true, false} { + home := t.TempDir() + xs := filepath.Join(home, "xs") + tc.setup(t, xs) + env := append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+xs, "TMPDIR="+home, "CODEX_SESSION_RELAY_STATE=") + args := []string{"--socket", filepath.Join(sockets, "app.sock"), "claim", "--event", "e"} + if tc.state { + state := filepath.Join(home, "state") + mustDo(t, os.MkdirAll(state, 0o700)) + py := exec.Command("uv", "run", "--no-sync", "python", "-c", "import sys;from codex_session_relay.store import Store;Store(sys.argv[1], socket_path=sys.argv[2]).close()", filepath.Join(state, "relay.sqlite3"), filepath.Join(sockets, "other.sock")) + py.Dir = filepath.Join(root, "packages", "codex-session-relay") + py.Env = env + mustDo(t, py.Run()) + args = append([]string{"--state", state}, args...) + } + var cmd *exec.Cmd + if python { + cmd = exec.Command("uv", append([]string{"run", "--no-sync", "codex-session-relay"}, args...)...) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + } else { + cmd = exec.Command(crwBinary(t), append([]string{"relay"}, args...)...) + } + cmd.Env = env + out, err := cmd.Output() + if e, ok := err.(*exec.ExitError); ok { + codes[i] = e.ExitCode() + } + side := &cliSide{home: home} + text := strings.ReplaceAll(side.normal(string(out)), sockets, "") + outs[i] = programPath.ReplaceAllString(text, `"$1 `) + } + if codes[0] != 2 || codes[1] != 2 || outs[0] != outs[1] { + t.Fatalf("exit python %d go %d\npython:\n%s\ngo:\n%s", codes[0], codes[1], outs[0], outs[1]) + } + }) + } +} + +// programPath is the program a recovery line names: each side names its own (Python its console +// script path, `crw relay` the argparse prog the multi-call entry passes, cli.ExecuteAs). +var programPath = regexp.MustCompile(`"(env -u CODEX_SESSION_RELAY_STATE )?(?:/\S*(?:/codex-session-relay|/crw)|'crw relay') `) diff --git a/internal/relay/delivery/service.go b/internal/relay/delivery/service.go new file mode 100644 index 00000000..7b21858a --- /dev/null +++ b/internal/relay/delivery/service.go @@ -0,0 +1,1095 @@ +package delivery + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/mergeturn" + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Journal kinds of a pre-send transition and an accepted settings note (CRW-235). +const ( + PresendWithheld = "delivery_presend_withheld" + SettingsNoted = "delivery_settings_noted" +) + +var claimable = []string{Queued, DeferredBusy, WithheldPreSend} + +var executionOnlyOutcomes = []string{"failed", "interrupted", "blocked_needs_input"} + +// LinkageReader is the linkage `up` reading delivery verifies its recipient against. Nil means +// the service copies the relationship row, as every caller written before the linkage does. +type LinkageReader interface { + Up(ctx context.Context, relationshipID string) (Obj, error) +} + +// Service is delivery.DeliveryService: one path to a transport call, through one atomic claim. +type Service struct { + Store *store.Store + Clock Clock + Policy RetryPolicy + RequireLifecycleEvidence bool + Linkage LinkageReader + RoleGate RoleGate + // RateLimited replaces the preflight when set (tests blind it to reach the claim's check). + RateLimited func(recipient string, now float64) bool +} + +// NewService builds a service with Python's defaults. +func NewService(s *store.Store, clock Clock) *Service { + return &Service{Store: s, Clock: clock, Policy: DefaultPolicy(), RequireLifecycleEvidence: true} +} + +// Get is DeliveryService.get. +func (d *Service) Get(ctx context.Context, eventID string) (Row, error) { + row, err := d.Find(ctx, eventID) + if err == nil && row == nil { + err = refuse(NotClaimable, "no delivery queued for event %s", store.PyRepr(eventID)) + } + return row, err +} + +// Find is DeliveryService.find. +func (d *Service) Find(ctx context.Context, eventID string) (Row, error) { + return one(ctx, d.Store, "SELECT * FROM deliveries WHERE event_id = ?", eventID) +} + +func (d *Service) eventRow(ctx context.Context, eventID string) (Row, error) { + return one(ctx, d.Store, "SELECT * FROM events WHERE event_id = ?", eventID) +} + +// Receipt is intake.get: the stored receipt record, or nil. +func (d *Service) Receipt(ctx context.Context, eventID string) (Obj, error) { + row, err := d.eventRow(ctx, eventID) + if err != nil || row == nil { + return nil, err + } + return loadsObj(row.S("receipt")), nil +} + +func manifestPaths(event Row) []string { + if event == nil { + return nil + } + receipt := loadsObj(event.S("receipt")) + entries, _ := get(receipt, "manifest") + list, _ := entries.([]any) + var paths []string + for _, e := range list { + if o, ok := e.(Obj); ok { + if p, ok := func() (string, bool) { v, _ := get(o, "path"); s, ok := v.(string); return s, ok }(); ok { + paths = append(paths, p) + } + } + } + return paths +} + +// ResolveRecipient is resolve_recipient: who a delivery goes to, verified against the linkage +// when one is wired, and the relationship row otherwise. +func (d *Service) ResolveRecipient(ctx context.Context, r Relationship, kind string) (string, Obj, error) { + frozen := r.Parent.TaskID + if kind == Revision { + frozen = r.Child.TaskID + } + if d.Linkage == nil { + return frozen, Obj{{Key: "source", Value: "relationship_row"}, {Key: "verified", Value: false}}, nil + } + rid := r.ID + reading, err := d.Linkage.Up(ctx, rid) + if err != nil { + return "", nil, err + } + if readable, _ := get(reading, "readable"); readable != true { + return "", nil, refuse(RelationUnreadable, "the linkage could not be read for relationship %s, so who owns its scope is unknown; the relationship row is not used as a fallback because an unreadable store has said nothing about the owner", store.PyRepr(rid)) + } + contention, _ := get(reading, "contention") + if state, _ := get(reading, "state"); state == "ambiguous" { + return "", nil, refuse(DuplicateScopeOwner, "the linkage reports more than one candidate for relationship %s; this reader will not choose between them: %s", store.PyRepr(rid), pyReprValue(contention)) + } + var live []any + items, _ := contention.([]any) + drifting := false + for _, item := range items { + o, _ := item.(Obj) + if c, _ := get(o, "contention"); truthy(c) { + live = append(live, item) + if c == "owner_drift" { + drifting = true + } + } + } + if len(live) > 0 { + reason := LinkConflict + if drifting { + reason = RelationOwnerDrift + } + return "", nil, refuse(reason, "the linkage reports the hierarchy of relationship %s as inconsistent, so who owns its scope is not settled: %s. A resolved state with contention is not a resolved owner, and delivery waits for the hierarchy to settle rather than picking the side that happens to match the frozen row", store.PyRepr(rid), pyReprValue(live)) + } + var wanted string + switch kind { + case Revision: + wanted = "issue" + case Completion, MergeTurnGrant: + wanted = "project" + default: + return "", nil, refuse(NotClaimable, "%s is not a delivery direction, so it has no resolvable recipient", store.PyRepr(kind)) + } + var level Obj + levels, _ := get(reading, "levels") + list, _ := levels.([]any) + for _, lv := range list { + o, _ := lv.(Obj) + if k, _ := get(o, "scopeKind"); k == wanted { + level = o + break + } + } + owner, _ := get(level, "owner") + if level == nil || owner == nil { + gaps, _ := get(reading, "gaps") + return "", nil, refuse(UnregisteredScope, "the linkage records no live %s owner for relationship %s; gaps %s. Nothing found is reported as nothing found, never as a delivery that may proceed", wanted, store.PyRepr(rid), pyReprValue(gaps)) + } + current := "" + var revision any + if o, ok := owner.(Obj); ok { + current = str(o, "taskId") + revision, _ = get(o, "revision") + } else { + current = pyStr(owner) + } + scopeKey, _ := get(level, "scopeKey") + if current != frozen { + return "", nil, refuse(RelationOwnerDrift, "relationship %s names %s but the linkage says %s %s is owned by %s. A report that arrived after the relationship changed is held rather than credited to either task; re-register the assignment under the current owner and deliver that", store.PyRepr(rid), store.PyRepr(frozen), wanted, pyReprValue(scopeKey), store.PyRepr(current)) + } + return current, Obj{{Key: "source", Value: "linkage"}, {Key: "verified", Value: true}, {Key: "scopeKind", Value: wanted}, {Key: "scopeKey", Value: scopeKey}, {Key: "revision", Value: revision}}, nil +} + +// Enqueue is DeliveryService.enqueue: idempotent, and only for a final event. recipient "" +// resolves the recipient. +func (d *Service) Enqueue(ctx context.Context, eventID, kind, recipient string) (Row, error) { + if kind == "" { + kind = Completion + } + event, err := d.eventRow(ctx, eventID) + if err != nil { + return nil, err + } + if event == nil { + return nil, refuse(NotClaimable, "event %s was never accepted", store.PyRepr(eventID)) + } + if event.S("stage") != "final" { + return nil, refuse(NotClaimable, "event %s is %s; only a final event may be delivered", store.PyRepr(eventID), store.PyRepr(event.S("stage"))) + } + rid := event.S("relationship_id") + relationship, err := RequireActive(ctx, d.Store, rid) + if err != nil { + return nil, err + } + var resolution Obj + if recipient == "" { + if recipient, resolution, err = d.ResolveRecipient(ctx, relationship, kind); err != nil { + return nil, err + } + } + if err := assertAssignmentDelivery(relationship, kind, recipient, nil, &rid, manifestPaths(event)); err != nil { + return nil, err + } + existing, err := d.Find(ctx, eventID) + if err != nil { + return nil, err + } + if existing != nil { + return existing, d.ClearIntent(ctx, eventID) + } + err = d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + if err := d.EnqueueIn(ctx, eventID, rid, kind, recipient); err != nil { + return err + } + _, err := execSQL(ctx, d.Store, "DELETE FROM delivery_intent WHERE event_id = ?", eventID) + return err + }) + if err != nil { + return nil, err + } + if verified, _ := get(resolution, "verified"); verified == true { + if err := d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + return journal(ctx, d.Store, "delivery_recipient_resolved", eventID, resolution, d.Clock.ISO()) + }); err != nil { + return nil, err + } + } + return d.Get(ctx, eventID) +} + +// EnqueueIn queues inside the caller's transaction (ctx must carry it). +func (d *Service) EnqueueIn(ctx context.Context, eventID, rid, kind, recipient string) error { + now := d.Clock.ISO() + if _, err := execSQL(ctx, d.Store, "INSERT OR IGNORE INTO deliveries (event_id, relationship_id, kind, recipient_task_id, recipient_thread_id, state, attempt_count, next_eligible_at, created_at, updated_at) VALUES (?,?,?,?,?,?,0,NULL,?,?)", eventID, rid, kind, recipient, recipient, Queued, now, now); err != nil { + return err + } + if err := journal(ctx, d.Store, "delivery_queued", eventID, Obj{{Key: "kind", Value: kind}}, now); err != nil { + return err + } + return d.AnnotatePredecessorsIn(ctx, eventID) +} + +// RecordIntentIn is record_intent_in: delivery was wanted here and refused for now. +func (d *Service) RecordIntentIn(ctx context.Context, eventID, rid, kind, recipient, errorText string, now float64) error { + row, err := one(ctx, d.Store, "SELECT attempts FROM delivery_intent WHERE event_id = ?", eventID) + if err != nil { + return err + } + attempts := int64(1) + if row != nil { + attempts = row.I("attempts") + 1 + } + _, err = execSQL(ctx, d.Store, "INSERT INTO delivery_intent (event_id, relationship_id, kind, recipient_task_id, attempts, next_retry_at, last_error, noted_at) VALUES (?,?,?,?,?,?,?,?) ON CONFLICT(event_id) DO UPDATE SET attempts = excluded.attempts, next_retry_at = excluded.next_retry_at, last_error = excluded.last_error", + eventID, rid, kind, recipient, attempts, now+d.backoff(attempts), errorText, d.Clock.ISO()) + return err +} + +func (d *Service) backoff(attempts int64) float64 { + base, ceiling := d.Policy.PresendBase, d.Policy.PresendMax + if base <= 0 { + return ceiling + } + steps := max(0, attempts-1) + if ceiling <= base || float64(steps) >= math.Ceil(math.Log2(ceiling/base)) { + return ceiling + } + return math.Min(ceiling, base*math.Pow(2, float64(steps))) +} + +// ClearIntent is clear_intent. +func (d *Service) ClearIntent(ctx context.Context, eventID string) error { + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := execSQL(ctx, d.Store, "DELETE FROM delivery_intent WHERE event_id = ?", eventID) + return err + }) +} + +const eligibleWhere = " WHERE d.state IN (?,?,?) AND d.hold_reason IS NULL AND (d.next_eligible_at IS NULL OR d.next_eligible_at <= ?) AND r.status = 'active' AND r.superseded_by IS NULL AND e.stage = 'final'" + +// EligibleParents is eligible_parents. +func (d *Service) EligibleParents(ctx context.Context, now float64) ([]string, error) { + rows, err := all(ctx, d.Store, "SELECT DISTINCT r.parent_task_id AS parent_task_id FROM deliveries d JOIN relationships r ON r.relationship_id = d.relationship_id JOIN events e ON e.event_id = d.event_id"+eligibleWhere+" ORDER BY r.parent_task_id", Queued, DeferredBusy, WithheldPreSend, now) + var out []string + for _, r := range rows { + out = append(out, r.S("parent_task_id")) + } + return out, err +} + +func (d *Service) eligibleForParent(ctx context.Context, parent string, now float64, limit, offset int) ([]Row, error) { + return all(ctx, d.Store, "SELECT d.*, r.parent_task_id AS parent_task_id FROM deliveries d JOIN relationships r ON r.relationship_id = d.relationship_id JOIN events e ON e.event_id = d.event_id"+eligibleWhere+" AND r.parent_task_id = ? ORDER BY d.created_at LIMIT ? OFFSET ?", Queued, DeferredBusy, WithheldPreSend, now, parent, limit, offset) +} + +// EligibleCount is eligible_count. +func (d *Service) EligibleCount(ctx context.Context, parent string, now float64) (int64, error) { + row, err := one(ctx, d.Store, "SELECT COUNT(*) AS c FROM deliveries d JOIN relationships r ON r.relationship_id = d.relationship_id JOIN events e ON e.event_id = d.event_id"+eligibleWhere+" AND r.parent_task_id = ?", Queued, DeferredBusy, WithheldPreSend, now, parent) + return row.I("c"), err +} + +// Eligible is eligible: every eligible parent, then a bounded share each, dealt one at a time. +func (d *Service) Eligible(ctx context.Context, now float64, limit, perParent, cursor int, offsets map[string]int) ([]Row, error) { + parents, err := d.EligibleParents(ctx, now) + if err != nil || len(parents) == 0 { + return nil, err + } + share := perParent + if share == 0 { + share = d.Policy.MaxSendsPerParentPerTick + } + start := cursor % len(parents) + order := append(slices.Clone(parents[start:]), parents[:start]...) + queues := map[string][]Row{} + for _, parent := range order { + offset := offsets[parent] + taken, err := d.eligibleForParent(ctx, parent, now, share, offset) + if err != nil { + return nil, err + } + if len(taken) < share && offset > 0 { + seen := map[string]bool{} + for _, r := range taken { + seen[r.S("event_id")] = true + } + again, err := d.eligibleForParent(ctx, parent, now, share, 0) + if err != nil { + return nil, err + } + for _, r := range again { + if len(taken) >= share { + break + } + if !seen[r.S("event_id")] { + taken = append(taken, r) + } + } + } + queues[parent] = taken + } + var selected []Row + for len(selected) < limit { + any := false + for _, parent := range order { + if len(selected) >= limit { + break + } + if len(queues[parent]) > 0 { + selected = append(selected, queues[parent][0]) + queues[parent] = queues[parent][1:] + any = true + } + } + if !any { + break + } + } + return selected, nil +} + +var errNotClaimable = errors.New("not claimable") +var errPaced = errors.New("paced") + +type superseded struct { + reason string + late bool +} + +func (s *superseded) Error() string { return "superseded: " + s.reason } + +// pacing reads send_pacing on the ctx-aware querier. +func (d *Service) pacing(ctx context.Context, recipient string, now float64) (Obj, error) { + window, earliest := d.Policy.RateWindows(now) + last, err := one(ctx, d.Store, "SELECT MAX(last_send_at) AS last FROM recipient_rate WHERE recipient_task_id = ? AND window_start BETWEEN ? AND ?", recipient, earliest, window) + if err != nil { + return nil, err + } + used, err := one(ctx, d.Store, "SELECT sends FROM recipient_rate WHERE recipient_task_id = ? AND window_start = ?", recipient, window) + if err != nil { + return nil, err + } + var lastAt *float64 + if last != nil && !last.N("last") { + v := last.F("last") + lastAt = &v + } + return d.Policy.Pacing(now, used.I("sends"), lastAt), nil +} + +// SendRefusal is send_refusal: why recipient may not be woken at now, or "". +func (d *Service) SendRefusal(ctx context.Context, recipient string, now float64) (string, error) { + p, err := d.pacing(ctx, recipient, now) + return str(p, "reason"), err +} + +// ReserveSend is reserve_send, inside the caller's claim transaction. +func (d *Service) ReserveSend(ctx context.Context, recipient string, now float64) (string, error) { + refused, err := d.SendRefusal(ctx, recipient, now) + if err != nil || refused != "" { + return refused, err + } + window := math.Floor(now/3600) * 3600 + _, err = execSQL(ctx, d.Store, "INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,1,?) ON CONFLICT(recipient_task_id, window_start) DO UPDATE SET sends = sends + 1, last_send_at = excluded.last_send_at", recipient, window, now) + return "", err +} + +func (d *Service) rateLimited(ctx context.Context, recipient string, now float64) (bool, error) { + if d.RateLimited != nil { + return d.RateLimited(recipient, now), nil + } + refused, err := d.SendRefusal(ctx, recipient, now) + return refused != "", err +} + +func (d *Service) pacedUntil(ctx context.Context, recipient string, now float64) (float64, error) { + p, err := d.pacing(ctx, recipient, now) + if err != nil { + return 0, err + } + if str(p, "reason") == HourlyCap { + reopens, _ := get(p, "reopensAt") + at, ok := reopens.(float64) + if !ok { + ws, _ := get(p, "windowStart") + at = ws.(float64) + RateWindowSeconds + } + return math.Min(at, now+60), nil + } + return now + d.Policy.MinSendInterval, nil +} + +// SupersessionReason is _supersession_reason, read on ctx's querier. "" means still current. +func (d *Service) SupersessionReason(ctx context.Context, eventID string) (string, error) { + event, err := one(ctx, d.Store, "SELECT relationship_id, execution_generation, outcome, event_id, receipt FROM events WHERE event_id = ?", eventID) + if err != nil || event == nil { + return "", err + } + if event.S("outcome") == MergeTurnGrant { + return mergeturn.GrantSupersessionFor(ctx, d.Store, event.S("receipt")) + } + rel, err := one(ctx, d.Store, "SELECT execution_generation FROM relationships WHERE relationship_id = ?", event.S("relationship_id")) + if err != nil || rel == nil { + return "", err + } + if event.I("execution_generation") < rel.I("execution_generation") { + return StaleGeneration, nil + } + if event.S("outcome") == Revision { + answered, err := one(ctx, d.Store, "SELECT 1 FROM events WHERE relationship_id = ? AND execution_generation = ? AND stage = 'final' AND suppressed_reason IS NULL AND event_id != ? AND outcome NOT IN ('merge_turn_grant')", event.S("relationship_id"), event.I("execution_generation"), eventID) + if err != nil || answered == nil { + return "", err + } + return SupersededRevision, nil + } + if slices.Contains(executionOnlyOutcomes, event.S("outcome")) { + return "", nil + } + head, err := HeadRevision(ctx, d.Store, event.S("relationship_id"), event.I("execution_generation")) + if err != nil { + return "", err + } + headID, _ := get(head, "eventId") + if headID == nil || headID == eventID { + return "", nil + } + successor, err := one(ctx, d.Store, "SELECT stage FROM events WHERE event_id = ?", headID) + if err != nil || successor == nil || successor.S("stage") != "final" { + return "", err + } + return SupersededRevision, nil +} + +func (d *Service) annotateIn(ctx context.Context, eventID, reason string) error { + _, err := execSQL(ctx, d.Store, "INSERT INTO delivery_supersession (event_id, reason, noted_at, applied) VALUES (?,?,?,0) ON CONFLICT(event_id) DO NOTHING", eventID, reason, d.Clock.ISO()) + return err +} + +// AnnotatePredecessorsIn marks outstanding deliveries this newly final event replaces. +func (d *Service) AnnotatePredecessorsIn(ctx context.Context, eventID string) error { + event, err := one(ctx, d.Store, "SELECT relationship_id, execution_generation FROM events WHERE event_id = ?", eventID) + if err != nil || event == nil { + return err + } + others, err := all(ctx, d.Store, "SELECT d.event_id FROM deliveries d JOIN events e ON e.event_id = d.event_id WHERE e.relationship_id = ? AND e.execution_generation = ? AND d.event_id != ? AND d.state IN ('queued','sending','held_uncertain','dispatched','inbox_only','deferred_busy','withheld_pre_send')", event.S("relationship_id"), event.I("execution_generation"), eventID) + if err != nil { + return err + } + for _, row := range others { + reason, err := d.SupersessionReason(ctx, row.S("event_id")) + if err != nil { + return err + } + if reason != "" { + if err := d.annotateIn(ctx, row.S("event_id"), reason); err != nil { + return err + } + } + } + return nil +} + +// AnnotatePredecessors is the same annotation in its own transaction. +func (d *Service) AnnotatePredecessors(ctx context.Context, eventID string) error { + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { return d.AnnotatePredecessorsIn(ctx, eventID) }) +} + +func (d *Service) supersedeIn(ctx context.Context, eventID, reason string) error { + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, hold_reason = ?, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND hold_reason IS NULL", Superseded, reason, d.Clock.ISO(), eventID, Queued, DeferredBusy, WithheldPreSend) + if err != nil { + return err + } + if changed == 1 { + return journal(ctx, d.Store, "delivery_superseded", eventID, Obj{{Key: "reason", Value: reason}}, d.Clock.ISO()) + } + return d.annotateIn(ctx, eventID, reason) +} + +// MarkSuperseded is mark_superseded: withdraw what the recipient cannot be acting on yet. +func (d *Service) MarkSuperseded(ctx context.Context, eventID, reason string) error { + if reason == "" { + reason = SupersededHold + } + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { return d.supersedeIn(ctx, eventID, reason) }) +} + +func (d *Service) suppressIfSuperseded(ctx context.Context, eventID string) (string, error) { + var reason string + err := d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + var err error + reason, err = d.SupersessionReason(ctx, eventID) + if err != nil || reason == "" { + return err + } + return d.supersedeIn(ctx, eventID, reason) + }) + return reason, err +} + +// ReadWorkReport reports whether a work report exists; the composed renderer is report.py's, +// ported by todo 24. Until then an event with a work report refuses to render. +func (d *Service) hasWorkReport(ctx context.Context, eventID string) (bool, error) { + row, err := one(ctx, d.Store, "SELECT 1 FROM work_reports WHERE event_id = ? LIMIT 1", eventID) + return row != nil, err +} + +func (d *Service) render(ctx context.Context, row Row, record Obj, request string) (string, error) { + report, err := d.hasWorkReport(ctx, row.S("event_id")) + if err != nil { + return "", err + } + if report { + return "", fmt.Errorf("%w: the composed report rendering (report.py) is owned by todo 24", ErrRendererNotPorted) + } + switch row.S("kind") { + case Revision: + return renderRevision(row, record, request), nil + case Completion: + return renderCompletion(row, record, request), nil + } + return "", fmt.Errorf("%w: the merge-turn grant notice is owned by the merge-turn port (todo 26)", ErrRendererNotPorted) +} + +// ErrRendererNotPorted marks a message whose bytes come from a renderer another todo ports; the +// message is refused rather than rendered differently. +var ErrRendererNotPorted = errors.New("delivery: renderer not ported") + +// Preview is preview_message for a reader that holds only a store (show --event --message). +// Reads use ctx, so a caller inside a transaction sees its writes. +func Preview(ctx context.Context, s *store.Store, eventID string) (string, error) { + return NewService(s, SystemClock{}).PreviewMessage(ctx, eventID) +} + +// PreviewMessage is preview_message / render_message: what the NEXT attempt would say. +func (d *Service) PreviewMessage(ctx context.Context, eventID string) (string, error) { + row, err := d.Get(ctx, eventID) + if err != nil { + return "", err + } + record, err := d.Receipt(ctx, eventID) + if err != nil { + return "", err + } + request, err := store.RequestID(eventID, int(row.I("attempt_count"))+1) + if err != nil { + return "", err + } + return d.render(ctx, row, record, request) +} + +type claimed struct { + attemptNo int64 + requestID string + message string +} + +// claim is _claim: authorization, staging and eligibility decided in one atomic statement, +// with the attempt, its bytes and the send budget written in the same transaction. +func (d *Service) claim(ctx context.Context, eventID string, now float64, owner, recipient string) (claimed, error) { + reason, err := d.suppressIfSuperseded(ctx, eventID) + if err != nil { + return claimed{}, err + } + if reason != "" { + return claimed{}, &superseded{reason: reason} + } + var out claimed + err = d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + late, err := d.SupersessionReason(ctx, eventID) + if err != nil { + return err + } + if late != "" { + return &superseded{reason: late, late: true} + } + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, lease_owner = ?, lease_until = ?, attempt_count = attempt_count + 1, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND hold_reason IS NULL AND (next_eligible_at IS NULL OR next_eligible_at <= ?) AND EXISTS (SELECT 1 FROM relationships r WHERE r.relationship_id = deliveries.relationship_id AND r.status = 'active' AND r.superseded_by IS NULL) AND EXISTS (SELECT 1 FROM events e WHERE e.event_id = deliveries.event_id AND e.stage = 'final') AND NOT EXISTS (SELECT 1 FROM events ev JOIN relationships rr ON rr.relationship_id = ev.relationship_id WHERE ev.event_id = deliveries.event_id AND ev.outcome NOT IN ('merge_turn_grant') AND ev.execution_generation < rr.execution_generation)", + Sending, owner, now+d.Policy.Lease, d.Clock.ISO(), eventID, Queued, DeferredBusy, WithheldPreSend, now) + if err != nil { + return err + } + if changed != 1 { + return errNotClaimable + } + row, err := d.Get(ctx, eventID) + if err != nil { + return err + } + out.attemptNo = row.I("attempt_count") + if out.requestID, err = store.RequestID(eventID, int(out.attemptNo)); err != nil { + return err + } + clash, err := one(ctx, d.Store, "SELECT event_id FROM attempts WHERE request_id = ?", out.requestID) + if err != nil { + return err + } + if clash != nil && clash.S("event_id") != eventID { + return refuse(NotClaimable, "request id %s already belongs to event %s", store.PyRepr(out.requestID), store.PyRepr(clash.S("event_id"))) + } + record, err := d.Receipt(ctx, eventID) + if err != nil { + return err + } + if out.message, err = d.render(ctx, row, record, out.requestID); err != nil { + return err + } + if _, err := execSQL(ctx, d.Store, "INSERT INTO attempts (request_id, event_id, attempt_no, kind, internal_state, state, sent_at, observed_at) VALUES (?,?,?,?,?,?,?,?)", out.requestID, eventID, out.attemptNo, row.S("kind"), "in_flight", HeldUncertain, d.Clock.ISO(), d.Clock.ISO()); err != nil { + return err + } + if _, err := execSQL(ctx, d.Store, "INSERT INTO attempt_messages (request_id, event_id, attempt_no, kind, message, rendered_at) VALUES (?,?,?,?,?,?)", out.requestID, eventID, out.attemptNo, row.S("kind"), out.message, d.Clock.ISO()); err != nil { + return err + } + if row.S("kind") == Revision { + if err := d.journalRestorationAttempted(ctx, eventID, record, out.attemptNo); err != nil { + return err + } + } + refused, err := d.ReserveSend(ctx, recipient, now) + if err != nil { + return err + } + if refused != "" { + return errPaced + } + return nil + }) + return out, err +} + +// journalRestorationAttempted records what the frozen bytes carried of a declared block +// (restoration.project_cap over the legacy renderer's cap). +func (d *Service) journalRestorationAttempted(ctx context.Context, eventID string, record Obj, attemptNo int64) error { + findingsValue, _ := get(record, "criteria") + findings, _ := findingsValue.([]any) + index := -1 + var block Obj + for i, f := range findings { + o, _ := f.(Obj) + if v, _ := get(o, "restoration"); truthy(v) { + index, block = i, o + break + } + } + if block == nil { + return nil + } + where := fmt.Sprintf("finding %d of %d", index+1, len(findings)) + outcome, detail := "carried", fmt.Sprintf("%s, within the %d this renderer shows", where, manifestLines) + if index >= manifestLines { + outcome, detail = "truncated", fmt.Sprintf("%s, past the %d this renderer shows", where, manifestLines) + } + id, _ := get(block, "id") + return journal(ctx, d.Store, "restoration_attempted", eventID, Obj{{Key: "outcome", Value: outcome}, {Key: "basis", Value: "relay-message/legacy"}, {Key: "criterion", Value: id}, {Key: "detail", Value: detail}, {Key: "attempt", Value: attemptNo}}, d.Clock.ISO()) +} + +func statusForRecord(l Lifecycle) string { + if l.Deliverable == "busy" { + return "active" + } + if s, ok := l.RuntimeStatus.(string); ok && slices.Contains([]string{"idle", "active", "notLoaded", "systemError"}, s) { + return s + } + return "unknown" +} + +// Attempt is DeliveryService.attempt. A nil record with a nil error is Python's None: nothing +// was due, or the delivery was deferred or withheld without an attempt. +func (d *Service) Attempt(ctx context.Context, eventID string, adapter Adapter, now *float64, owner string) (Obj, error) { + at := d.Clock.Now() + if now != nil { + at = *now + } + if owner == "" { + owner = "relay" + } + row, err := d.Get(ctx, eventID) + if err != nil { + return nil, err + } + relationship, err := LoadRelationship(ctx, d.Store, row.S("relationship_id")) + if err != nil { + return nil, err + } + recipient := row.S("recipient_task_id") + if !row.N("hold_reason") && row.S("hold_reason") != "" { + return nil, nil + } + if !slices.Contains(claimable, row.S("state")) { + return nil, nil + } + if relationship.Status != "active" && relationship.SupersededBy == nil { + return d.WithholdInactive(ctx, eventID, relationship, at, row.I("attempt_count")) + } + if !row.N("next_eligible_at") && row.F("next_eligible_at") > at { + return nil, nil + } + event, err := d.eventRow(ctx, eventID) + if err != nil { + return nil, err + } + eventRID := event.S("relationship_id") + thread := row.S("recipient_thread_id") + if err := assertAssignmentDelivery(relationship, row.S("kind"), recipient, &thread, &eventRID, manifestPaths(event)); err != nil { + return nil, err + } + limited, err := d.rateLimited(ctx, recipient, at) + if err != nil { + return nil, err + } + if limited { + when, err := d.pacedUntil(ctx, recipient, at) + if err != nil { + return nil, err + } + return nil, d.reschedule(ctx, eventID, row.S("state"), when, row.I("attempt_count")) + } + var cwd any + if row.S("kind") == Completion || row.S("kind") == MergeTurnGrant { + cwd = relationship.Parent.Cwd + } + observation := Observe(adapter, thread, cwd, d.RequireLifecycleEvidence) + if err := RecordLifecycle(ctx, d.Store, d.Clock, observation); err != nil { + return nil, err + } + if observation.IsBusy() { + return nil, d.deferBusy(ctx, eventID, row, at) + } + if !observation.MaySend() { + return nil, d.withhold(ctx, eventID, observation, at, row, relationship) + } + settings, err := AuthorizedSettings(ctx, d.Store, recipient, d.RoleGate) + if Reason(err) != "" { + return nil, d.withholdSettings(ctx, eventID, at, err, row) + } + if err != nil { + return nil, err + } + known, err := adapter.ListTurnIDs(thread, 25) + if err != nil { + return nil, err + } + c, err := d.claim(ctx, eventID, at, owner, recipient) + var gone *superseded + switch { + case errors.Is(err, errPaced): + when, err := d.pacedUntil(ctx, recipient, at) + if err != nil { + return nil, err + } + return nil, d.reschedule(ctx, eventID, row.S("state"), when, row.I("attempt_count")) + case errors.Is(err, errNotClaimable): + return nil, nil + case errors.As(err, &gone): + if gone.late { + if _, err := d.suppressIfSuperseded(ctx, eventID); err != nil { + return nil, err + } + } + return Obj{{Key: "deliveryState", Value: Superseded}, {Key: "supersededReason", Value: gone.reason}, {Key: "eventId", Value: eventID}, {Key: "sendAttempted", Value: "no"}}, nil + case err != nil: + return nil, err + } + receipt, sendErr := adapter.SendMessage(c.requestID, thread, c.message, settings) + if sendErr != nil { + receipt = Obj{{Key: "requestId", Value: c.requestID}, {Key: "status", Value: OutcomeUnknown}, {Key: "error", Value: errorLabel(sendErr)}} + } + facts := Classify(receipt) + findings, _ := get(receipt, "settingsFindings") + if facts.FailedOperation != nil || slices.Contains([]string{WithheldPreSend, InboxOnly, HeldUncertain}, facts.DeliveryState) { + operation := "transport" + if op, ok := facts.FailedOperation.(string); ok { + operation = op + } + if truthy(findings) { + operation = "settings_check" + } + detail := facts.DeliveryState + if t, ok := facts.ErrorText.(string); ok && t != "" { + detail = t + } + safe := facts.RetrySafe + if err := d.RecordFailure(ctx, eventID, operation, detail, row.S("relationship_id"), relationship.Parent.TaskID, facts.RPCErrorCode, renderFindings(findings), &safe, nil); err != nil { + return nil, err + } + } + previously := false + if t, ok := facts.TurnID.(string); ok && slices.Contains(known, t) { + previously = true + } + record, err := AttemptRecord(facts, c.requestID, eventID, c.attemptNo, recipient, statusForRecord(observation), d.Clock.ISO(), nil) + if err != nil { + return nil, err + } + notes, _ := get(receipt, "settingsNotes") + elsewhere, err := d.settle(ctx, eventID, c.requestID, record, facts, previously, at, settingsRefusalOf(facts, findings), notes) + if err != nil { + return nil, err + } + if elsewhere != nil { + result := record + if !elsewhere.N("record") { + result = loadsObj(elsewhere.S("record")) + } + result = append(append(Obj(nil), result...), F{Key: "_settledElsewhere", Value: true}, F{Key: "_deliveryState", Value: elsewhere.Opt("delivery_state")}, F{Key: "_lifecycle", Value: observation.Deliverable}) + return result, nil + } + origin := any(nil) + if previously { + origin = "steered_observed_turn" + } else if facts.TurnID != nil { + origin = "unknown" + } + result := append(append(Obj(nil), record...), F{Key: "_turnPreviouslyObserved", Value: previously}, F{Key: "_turnOrigin", Value: origin}, F{Key: "_lifecycle", Value: observation.Deliverable}) + return result, nil +} + +func renderFindings(findings any) any { + list, _ := findings.([]any) + var parts []string + for _, f := range list { + o, ok := f.(Obj) + if !ok { + continue + } + field, present := get(o, "field") + if !present { + if field, present = get(o, "code"); !present { + field = "?" + } + } + expected, _ := get(o, "expected") + returned, _ := get(o, "returned") + parts = append(parts, fmt.Sprintf("%s: expected %s, host %s", pyStr(field), pyReprValue(expected), pyReprValue(returned))) + } + if len(parts) == 0 { + return nil + } + return strings.Join(parts, "; ") +} + +var attemptSettingsCodes = append(slices.Clone(SettingsRefusals), UnsupportedApprovalPolicy) + +// settingsRefusalOf is settings_refusal_of: this attempt's settings cause, or nil. +func settingsRefusalOf(facts Facts, findings any) any { + code, ok := facts.RPCErrorCode.(string) + if facts.FailedOperation != "thread/resume" || !ok || !slices.Contains(attemptSettingsCodes, code) { + return nil + } + var field any + if list, _ := findings.([]any); len(list) > 0 { + if o, ok := list[0].(Obj); ok { + if f, ok := get(o, "field"); ok { + if s, ok := f.(string); ok { + field = s + } + } + } + } + return Obj{{Key: "reason", Value: code}, {Key: "field", Value: field}} +} + +// RecordFailure is record_failure. +func (d *Service) RecordFailure(ctx context.Context, scope, operation, detail string, rid, parent, code, difference any, retrySafe *bool, nextRetry any) error { + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + return d.recordFailureIn(ctx, scope, operation, d.Clock.ISO(), detail, rid, parent, code, difference, retrySafe, nextRetry) + }) +} + +func (d *Service) recordFailureIn(ctx context.Context, scope, operation, occurred, detail string, rid, parent, code, difference any, retrySafe *bool, nextRetry any) error { + var safe any + if retrySafe != nil { + safe = int64(0) + if *retrySafe { + safe = int64(1) + } + } + _, err := execSQL(ctx, d.Store, "INSERT INTO failed_operations (scope_key, operation, relationship_id, parent_task_id, detail, error_code, difference, retry_safe, occurred_at, next_retry_at) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(scope_key, operation) DO UPDATE SET detail=excluded.detail, error_code=excluded.error_code, difference=excluded.difference, retry_safe=excluded.retry_safe, occurred_at=excluded.occurred_at, next_retry_at=excluded.next_retry_at, relationship_id=COALESCE(excluded.relationship_id, failed_operations.relationship_id), parent_task_id=COALESCE(excluded.parent_task_id, failed_operations.parent_task_id)", + scope, operation, nullable(rid), nullable(parent), detail, code, difference, safe, occurred, nextRetry) + return err +} + +func (d *Service) reschedule(ctx context.Context, eventID, state string, when float64, attempts int64) error { + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND attempt_count = ?", state, when, d.Clock.ISO(), eventID, Queued, DeferredBusy, WithheldPreSend, attempts) + return err + }) +} + +// Reschedule is _reschedule (tests reach it as Python's do). +func (d *Service) Reschedule(ctx context.Context, eventID, state string, when float64, attempts int64) error { + return d.reschedule(ctx, eventID, state, when, attempts) +} + +// DeferBusy is _defer_busy: a busy recipient is left alone and retried later. +func (d *Service) DeferBusy(ctx context.Context, eventID string, row Row, now float64) error { + return d.deferBusy(ctx, eventID, row, now) +} + +func (d *Service) deferBusy(ctx context.Context, eventID string, row Row, now float64) error { + attempts := row.I("attempt_count") + var hold any + if attempts >= d.Policy.BusyMaxAttempts { + hold = d.Policy.CapReason("busy") + } + when := now + d.Policy.DelayFor(attempts+1, "busy") + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + stamp := d.Clock.ISO() + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, hold_reason = ?, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND attempt_count = ?", DeferredBusy, when, hold, stamp, eventID, Queued, DeferredBusy, WithheldPreSend, attempts) + if err != nil { + return err + } + if changed == 1 { + if err := d.recordFailureIn(ctx, eventID, "parent_busy", stamp, "the recipient is mid-turn and is never interrupted", row.S("relationship_id"), nil, nil, nil, nil, when); err != nil { + return err + } + } + if row.S("state") != DeferredBusy { + return journal(ctx, d.Store, "delivery_deferred_busy", eventID, "", d.Clock.ISO()) + } + return nil + }) +} + +// WithholdInactive is _withhold_inactive: a status a person set stops the delivery with the +// reason kept; nil when the relationship moved under the read. +func (d *Service) WithholdInactive(ctx context.Context, eventID string, r Relationship, now float64, attempts int64) (Obj, error) { + when := now + d.Policy.LifecycleRecheck + moved := false + err := d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = MAX(?, COALESCE(next_eligible_at, 0)), updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND attempt_count = ? AND EXISTS (SELECT 1 FROM relationships r WHERE r.relationship_id = deliveries.relationship_id AND r.status = ? AND r.superseded_by IS NULL)", + WithheldPreSend, when, d.Clock.ISO(), eventID, Queued, DeferredBusy, WithheldPreSend, attempts, r.Status) + if err != nil { + return err + } + if changed != 1 { + moved = true + return nil + } + if err := journal(ctx, d.Store, "delivery_withheld_inactive", eventID, Obj{{Key: "relationshipId", Value: r.ID}, {Key: "status", Value: r.Status}}, d.Clock.ISO()); err != nil { + return err + } + return journal(ctx, d.Store, PresendWithheld, eventID, Obj{{Key: "reason", Value: RelationshipNotActive}, {Key: "operation", Value: nil}}, d.Clock.ISO()) + }) + if err != nil || moved { + return nil, err + } + return Obj{{Key: "deliveryState", Value: WithheldPreSend}, {Key: "withheldReason", Value: RelationshipNotActive}, {Key: "relationshipStatus", Value: r.Status}, {Key: "eventId", Value: eventID}, {Key: "sendAttempted", Value: "no"}}, nil +} + +func (d *Service) withhold(ctx context.Context, eventID string, l Lifecycle, now float64, row Row, r Relationship) error { + when := now + d.Policy.LifecycleRecheck + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + stamp := d.Clock.ISO() + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND attempt_count = ?", WithheldPreSend, when, stamp, eventID, Queued, DeferredBusy, WithheldPreSend, row.I("attempt_count")) + if err != nil { + return err + } + if changed == 1 { + detail := l.Detail + if detail == "" { + detail = pyStr(l.WithholdReason) + if l.WithholdReason == nil { + detail = "not deliverable" + } + } + if err := d.recordFailureIn(ctx, eventID, "lifecycle_read", stamp, detail, row.S("relationship_id"), r.Parent.TaskID, l.WithholdReason, nil, nil, when); err != nil { + return err + } + if err := journal(ctx, d.Store, PresendWithheld, eventID, Obj{{Key: "reason", Value: l.WithholdReason}, {Key: "operation", Value: "lifecycle_read"}}, stamp); err != nil { + return err + } + } + return journal(ctx, d.Store, "delivery_withheld", eventID, Obj{{Key: "reason", Value: l.WithholdReason}, {Key: "detail", Value: l.Detail}}, d.Clock.ISO()) + }) +} + +func (d *Service) withholdSettings(ctx context.Context, eventID string, now float64, refusal error, row Row) error { + when := now + d.Policy.LifecycleRecheck + reason := Reason(refusal) + if reason == "" { + reason = "settings_unavailable" + } + detail := Detail(refusal) + return d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + stamp := d.Clock.ISO() + changed, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, updated_at = ? WHERE event_id = ? AND state IN (?,?,?) AND attempt_count = ?", WithheldPreSend, when, stamp, eventID, Queued, DeferredBusy, WithheldPreSend, row.I("attempt_count")) + if err != nil { + return err + } + if err := journal(ctx, d.Store, "delivery_withheld", eventID, Obj{{Key: "reason", Value: reason}, {Key: "detail", Value: detail}}, d.Clock.ISO()); err != nil { + return err + } + if changed != 1 { + return nil + } + safe := true + if err := d.recordFailureIn(ctx, eventID, "settings_check", stamp, detail, row.S("relationship_id"), nil, reason, nil, &safe, when); err != nil { + return err + } + return journal(ctx, d.Store, PresendWithheld, eventID, Obj{{Key: "reason", Value: reason}, {Key: "operation", Value: "settings_check"}, {Key: "detail", Value: detail}}, stamp) + }) +} + +// settle is _settle: this send's own attempt, if nothing else settled it first. A non-nil row +// is what another reader stored. +func (d *Service) settle(ctx context.Context, eventID, requestID string, record Obj, facts Facts, previously bool, now float64, settingsRefusal any, notes any) (Row, error) { + if err := AssertAttemptInvariants(record); err != nil { + return nil, err + } + state := facts.DeliveryState + attemptNo, _ := get(record, "attemptNo") + n := attemptNo.(int64) + var hold, when any + switch state { + case DeferredBusy: + when = now + d.Policy.DelayFor(n+1, "busy") + if n >= d.Policy.BusyMaxAttempts { + hold = d.Policy.CapReason("busy") + } + case WithheldPreSend: + when = now + d.Policy.DelayFor(n+1, "presend") + if n >= d.Policy.MaxAttempts { + hold = d.Policy.CapReason("presend") + } + case InboxOnly: + hold = PushChannelClosed + } + var stored Row + err := d.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + settled, err := execSQL(ctx, d.Store, "UPDATE attempts SET internal_state = 'settled', state = ?, record = ?, observed_at = ? WHERE request_id = ? AND internal_state = 'in_flight'", state, dumps(record), str(record, "observedAt"), requestID) + if err != nil { + return err + } + if settled != 1 { + stored, err = one(ctx, d.Store, "SELECT a.record, a.state, a.affirmative_evidence, d.state AS delivery_state FROM attempts a JOIN deliveries d ON d.event_id = a.event_id WHERE a.request_id = ?", requestID) + if err != nil { + return err + } + detail := Obj{{Key: "requestId", Value: requestID}, {Key: "receiptState", Value: state}, {Key: "storedState", Value: stored.Opt("state")}, {Key: "storedEvidence", Value: stored.Opt("affirmative_evidence")}} + return journal(ctx, d.Store, "delivery_attempt_settled_elsewhere", eventID, detail, d.Clock.ISO()) + } + var evidence any + if state == Dispatched { + evidence = "transport_accepted" + } + if _, err := execSQL(ctx, d.Store, "UPDATE deliveries SET state = ?, next_eligible_at = ?, hold_reason = ?, dispatch_evidence = ?, dispatch_turn_id = ?, lease_owner = NULL, lease_until = NULL, updated_at = ? WHERE event_id = ?", state, when, hold, evidence, facts.TurnID, d.Clock.ISO(), eventID); err != nil { + return err + } + safe, _ := get(record, "retrySafe") + if err := journal(ctx, d.Store, "delivery_attempted", eventID, Obj{{Key: "requestId", Value: requestID}, {Key: "state", Value: state}, {Key: "retrySafe", Value: safe}, {Key: "turnPreviouslyObserved", Value: previously}, {Key: "hold", Value: hold}, {Key: "settingsRefusal", Value: settingsRefusal}}, d.Clock.ISO()); err != nil { + return err + } + if truthy(notes) { + return journal(ctx, d.Store, SettingsNoted, eventID, Obj{{Key: "requestId", Value: requestID}, {Key: "notes", Value: notes}}, d.Clock.ISO()) + } + return nil + }) + return stored, err +} diff --git a/internal/relay/delivery/services_test.go b/internal/relay/delivery/services_test.go new file mode 100644 index 00000000..5da9f55f --- /dev/null +++ b/internal/relay/delivery/services_test.go @@ -0,0 +1,47 @@ +package delivery + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// VCU-13: closing the CLI's services releases what they opened and builds nothing just to +// close it. The Go CLI owns one store per invocation and no host adapter until one is needed. +func TestVCU13_the_cli_closes_what_it_opened_and_builds_nothing_to_close(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "xs")) + state := filepath.Join(home, "state") + var out, errw bytes.Buffer + + code, handled := ExecuteCLI(context.Background(), []string{"--state", state, "ack-proof", "--event", "0123456789abcdef0123456789abcdef", "--turn", "t"}, &out, &errw) + if !handled || code != 0 { + t.Fatalf("ack-proof %d %s", code, out.String()) + } + if _, err := os.Stat(filepath.Join(state, "relay.sqlite3")); !os.IsNotExist(err) { + t.Fatal("a command that needs no store built one") + } + out.Reset() + code, _ = ExecuteCLI(context.Background(), []string{"--state", state, "deliver"}, &out, &errw) + if code != 4 { + t.Fatalf("deliver without a socket is a usage error, got %d", code) + } + out.Reset() + code, _ = ExecuteCLI(context.Background(), []string{"--state", state, "criteria-register", "--relationship", "r", "--criterion", "c1=one"}, &out, &errw) + if code != 0 { + t.Fatalf("criteria-register %d %s", code, out.String()) + } + // The store the command opened is closed: a writer with no busy wait takes the lock at once. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + s, err := store.Open(ctx, filepath.Join(state, "relay.sqlite3"), "") + mustDo(t, err) + defer s.Close() + mustDo(t, s.AppendJournal(ctx, store.JournalEntry{At: "t", Kind: "probe", Subject: "s", Detail: "{}"})) +} diff --git a/internal/relay/delivery/settings.go b/internal/relay/delivery/settings.go new file mode 100644 index 00000000..1c317a37 --- /dev/null +++ b/internal/relay/delivery/settings.go @@ -0,0 +1,345 @@ +package delivery + +import ( + "context" + "fmt" + "slices" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +var requiredSettings = []string{"sandbox", "approvalPolicy", "cwd", "runtimeWorkspaceRoots", "model", "reasoningEffort", "environments"} + +// CarriedApprovalPolicies are the approval policies this transport carries (settings.py). +var CarriedApprovalPolicies = []string{"never", "on-request"} + +var resumeSandboxMode = map[string]string{"workspaceWrite": "workspace-write", "readOnly": "read-only", "dangerFullAccess": "danger-full-access"} + +// TaskSettings is settings.TaskSettings: the recorded execution settings a send preserves. +type TaskSettings struct { + Data Obj + SettingsFreeResume bool +} + +func (t TaskSettings) missing() []string { + var absent []string + for _, field := range requiredSettings { + v, ok := get(t.Data, field) + if !ok || v == nil { + absent = append(absent, field) + } + } + return absent +} + +func textList(v any) bool { + a, ok := v.([]any) + if !ok { + return false + } + for _, item := range a { + if _, ok := item.(string); !ok { + return false + } + } + return true +} + +func pyTypeName(v any) string { + switch v.(type) { + case nil: + return "NoneType" + case bool: + return "bool" + case string: + return "str" + case int64: + return "int" + case float64: + return "float" + case Obj: + return "dict" + case []any: + return "list" + } + return fmt.Sprintf("%T", v) +} + +func shapeOf(v any) string { + if a, ok := v.([]any); ok { + for _, one := range a { + if _, ok := one.(string); !ok { + if one == nil { + return "a list holding None" + } + return "a list holding " + pyTypeName(one) + } + } + return "a list of str" + } + if v == nil { + return "absent" + } + return pyTypeName(v) +} + +func environmentsProblem(v any) (string, string, bool) { + a, ok := v.([]any) + if !ok { + return "", "is " + shapeOf(v) + ", not a list of environment objects", true + } + for i, entry := range a { + o, ok := entry.(Obj) + if !ok { + return fmt.Sprintf("[%d]", i), "is " + shapeOf(entry) + ", not an object", true + } + for _, key := range []string{"environmentId", "cwd"} { + value, _ := get(o, key) + if _, ok := value.(string); !ok { + return fmt.Sprintf("[%d].%s", i, key), "is " + shapeOf(value) + ", not str", true + } + } + if roots, present := get(o, "runtimeWorkspaceRoots"); present && !textList(roots) { + return fmt.Sprintf("[%d].runtimeWorkspaceRoots", i), "is " + shapeOf(roots) + ", not a list of str", true + } + } + return "", "", false +} + +func (t TaskSettings) mistyped() []string { + var wrong []string + for _, field := range []string{"cwd", "model", "reasoningEffort"} { + if v, _ := get(t.Data, field); pyTypeName(v) != "str" { + wrong = append(wrong, field) + } + } + if v, _ := get(t.Data, "runtimeWorkspaceRoots"); !textList(v) { + wrong = append(wrong, "runtimeWorkspaceRoots") + } + if v, _ := get(t.Data, "environments"); func() bool { _, _, bad := environmentsProblem(v); return bad }() { + wrong = append(wrong, "environments") + } + return wrong +} + +func (t TaskSettings) mistypedDetail(field string) string { + value, _ := get(t.Data, field) + switch field { + case "runtimeWorkspaceRoots": + return "runtimeWorkspaceRoots is " + shapeOf(value) + ", not a list of str" + case "environments": + where, what, _ := environmentsProblem(value) + return "environments" + where + " " + what + } + return field + " is " + pyTypeName(value) + ", not str" +} + +func (t TaskSettings) sandboxMode() (string, bool) { + policy, ok := get(t.Data, "sandbox") + o, isObj := policy.(Obj) + if !ok || !isObj { + return "", false + } + kind, isText := get(o, "type") + name, _ := kind.(string) + if !isText || name == "" { + if _, s := kind.(string); !s { + return "", false + } + } + mode, known := resumeSandboxMode[name] + return mode, known +} + +var policyDefaults = map[string]Obj{ + "workspaceWrite": {{Key: "writableRoots", Value: []any{}}, {Key: "networkAccess", Value: false}, {Key: "excludeTmpdirEnvVar", Value: false}, {Key: "excludeSlashTmp", Value: false}}, + "readOnly": {{Key: "networkAccess", Value: false}}, + "externalSandbox": {{Key: "networkAccess", Value: "restricted"}}, + "dangerFullAccess": {}, +} + +// normalisePolicy is settings.normalise_policy's readability decision (nil when unreadable). +func normalisePolicy(policy any) Obj { + o, ok := policy.(Obj) + if !ok { + return nil + } + kind, ok := get(o, "type") + name, isText := kind.(string) + if !ok || !isText { + return nil + } + declared := policyDefaults[name] + merged := append(Obj(nil), declared...) + for _, f := range o { + if f.Key != "type" { + merged = set(merged, f.Key, f.Value) + } + } + merged = set(merged, "type", name) + for _, d := range declared { + value, _ := get(merged, d.Key) + switch d.Value.(type) { + case bool: + if _, ok := value.(bool); !ok { + return nil + } + case []any: + if !textList(value) { + return nil + } + default: + if pyTypeName(value) != pyTypeName(d.Value) { + return nil + } + } + } + if roots, ok := get(merged, "writableRoots"); ok && !textList(roots) { + return nil + } + return merged +} + +func pyReprValue(v any) string { + switch t := v.(type) { + case nil: + return "None" + case bool: + if t { + return "True" + } + return "False" + case string: + return store.PyRepr(t) + case int64: + return fmt.Sprint(t) + case float64: + return pyFloat(t) + case []any: + parts := make([]string, len(t)) + for i, x := range t { + parts[i] = pyReprValue(x) + } + return "[" + strings.Join(parts, ", ") + "]" + case Obj: + parts := make([]string, len(t)) + for i, f := range t { + parts[i] = store.PyRepr(f.Key) + ": " + pyReprValue(f.Value) + } + return "{" + strings.Join(parts, ", ") + "}" + } + return fmt.Sprint(v) +} + +// RequireUsable is TaskSettings.require_usable: shape before meaning, every offender named. +func (t TaskSettings) RequireUsable() error { + if absent := t.missing(); len(absent) > 0 { + if env, ok := get(t.Data, "environments"); ok { + if _, isList := env.([]any); isList { + absent = slices.DeleteFunc(absent, func(s string) bool { return s == "environments" }) + } + } + if len(absent) > 0 { + return refuse(SettingsIncomplete, "missing %s", strings.Join(absent, ", ")) + } + } + if wrong := t.mistyped(); len(wrong) > 0 { + details := make([]string, len(wrong)) + for i, field := range wrong { + details[i] = t.mistypedDetail(field) + } + return refuse(SettingsMistyped, "%s", strings.Join(details, "; ")) + } + policy, _ := get(t.Data, "approvalPolicy") + if p, ok := policy.(string); !ok || !slices.Contains(CarriedApprovalPolicies, p) { + return refuse(UnsupportedApprovalPolicy, "the recorded approvalPolicy is %s; this transport carries only 'never' and 'on-request', leaving every approval a turn raises with the thread's own approver", pyReprValue(policy)) + } + if _, ok := t.sandboxMode(); !ok { + recorded, _ := get(t.Data, "sandbox") + o, isObj := recorded.(Obj) + if !isObj { + return refuse(UnsupportedSandboxType, "the recorded sandbox is %s, not the policy object a creation result reports, so it does not record the full policy a resume would have to restore", pyTypeName(recorded)) + } + kind, _ := get(o, "type") + return refuse(UnsupportedSandboxType, "%s has no ThreadResumeParams.sandbox mode, so it cannot be restored on a resume", pyReprValue(kind)) + } + sandbox, _ := get(t.Data, "sandbox") + if normalisePolicy(sandbox) == nil { + return refuse(UnsupportedSandboxType, "the recorded sandbox policy cannot be read in full, so no response could confirm it") + } + return nil +} + +// RoleGate decides the role-policy half of authorized_settings for a task bound to a scope. +// The registry track owns rolepolicy (todo 25); until it is wired, a bound task is refused as +// Python refuses it when this process has no readable policy. +type RoleGate func(ctx context.Context, s *store.Store, taskID string, settings *TaskSettings) error + +// DefaultRoleGate is bound_role + the unresolved-policy refusal. +func DefaultRoleGate(ctx context.Context, s *store.Store, taskID string, _ *TaskSettings) error { + rows, err := all(ctx, s, "SELECT DISTINCT role FROM scope_bindings WHERE task_id = ? AND status IN (?,?) AND superseded_by IS NULL", taskID, "active", "paused") + if err != nil { + return err + } + if len(rows) == 0 { + return nil + } + if len(rows) > 1 { + roles := make([]string, len(rows)) + for i, r := range rows { + roles[i] = r.S("role") + } + slices.Sort(roles) + return refuse(RoleBindingMismatch, "%s holds live bindings at %s, and one task holds one role. Nothing was sent and no turn was started, because checking its authorization against either of them would report a clean answer derived from an arbitrary choice. Resolve the bindings first.", store.PyRepr(taskID), reprList(roles)) + } + return refuse(RolePolicyUnconfigured, "%s is bound as %s and this process cannot read a role policy to check its authorization against: CODEX_THREAD_BRIDGE_EXECUTION_POLICY is not set in this process, so no role policy can be read. Nothing was sent and no turn was started. Set the policy for this process and the held deliveries resume on the next pass.", store.PyRepr(taskID), store.PyRepr(rows[0].S("role"))) +} + +// AuthorizedSettings is delivery.authorized_settings. +func AuthorizedSettings(ctx context.Context, s *store.Store, taskID string, gate RoleGate) (*TaskSettings, error) { + row, err := one(ctx, s, "SELECT settings FROM authorized_settings WHERE task_id = ?", taskID) + if err != nil { + return nil, err + } + if row == nil { + return nil, refuse(SettingsUnavailable, "no authorized settings recorded for %s; register them from the creation result before a send can preserve them", store.PyRepr(taskID)) + } + data := loadsObj(row.S("settings")) + settings := &TaskSettings{Data: data} + if err := settings.RequireUsable(); err != nil { + return nil, err + } + if gate == nil { + gate = DefaultRoleGate + } + if err := gate(ctx, s, taskID, settings); err != nil { + return nil, err + } + return settings, nil +} + +var policyConfigKeys = map[string][][3]string{ + "workspaceWrite": {{"writableRoots", "sandbox_workspace_write", "writable_roots"}, {"networkAccess", "sandbox_workspace_write", "network_access"}, {"excludeTmpdirEnvVar", "sandbox_workspace_write", "exclude_tmpdir_env_var"}, {"excludeSlashTmp", "sandbox_workspace_write", "exclude_slash_tmp"}}, +} + +// ResumeParams is TaskSettings.resume_params: only fields ThreadResumeParams defines, and never +// an approvalPolicy - the host would apply it to a thread the resume loads. +func (t TaskSettings) ResumeParams(thread string) Obj { + mode, _ := t.sandboxMode() + roots, _ := get(t.Data, "runtimeWorkspaceRoots") + cwd, _ := get(t.Data, "cwd") + model, _ := get(t.Data, "model") + effort, _ := get(t.Data, "reasoningEffort") + config := Obj{{Key: "model_reasoning_effort", Value: effort}} + sandbox, _ := get(t.Data, "sandbox") + policy := normalisePolicy(sandbox) + for _, k := range policyConfigKeys[str(policy, "type")] { + if v, ok := get(policy, k[0]); ok { + section, _ := get(config, k[1]) + o, _ := section.(Obj) + config = set(config, k[1], set(o, k[2], v)) + } + } + return Obj{{Key: "threadId", Value: thread}, {Key: "excludeTurns", Value: true}, {Key: "sandbox", Value: mode}, {Key: "cwd", Value: cwd}, {Key: "runtimeWorkspaceRoots", Value: roots}, {Key: "model", Value: model}, {Key: "config", Value: config}} +} diff --git a/internal/relay/delivery/snapshot.go b/internal/relay/delivery/snapshot.go new file mode 100644 index 00000000..0675ad59 --- /dev/null +++ b/internal/relay/delivery/snapshot.go @@ -0,0 +1,219 @@ +package delivery + +import ( + "context" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/mergeturn" +) + +// reportedState is _reported_state: what an operator should read, beside the raw state. +func reportedState(row Row, ack Row, grant string, superseded Row) string { + if ack != nil && ack.S("verified") == "verified" && ack.I("accepted") != 0 { + return "acknowledged" + } + if row.S("kind") == MergeTurnGrant { + switch { + case grant == mergeturn.GrantAnswered: + return "grant_acknowledged" + case grant != "": + return "superseded:" + grant + case row.S("state") == Dispatched: + return "dispatched_awaiting_grant_acknowledgement" + } + } + state := row.S("state") + hold := row.S("hold_reason") + switch { + case state == InboxOnly: + return "stored_not_woken" + case state == Dispatched: + return "dispatched_awaiting_ack" + case state == HeldUncertain: + if superseded != nil { + return "superseded:" + superseded.S("reason") + } + if hold != "" { + return "held:" + hold + } + return "held_uncertain_awaiting_evidence" + case state == Queued && row.S("dispatch_evidence") == HostLostTurn && hold == "": + return "redelivering:" + HostLostTurn + case hold != "": + return "held:" + hold + } + return state +} + +// phase is _phase for the kinds this package delivers (the settings-hold reading belongs to +// todos 25/22). +func phase(row Row, attempts []Row, ack, failure Row, grant string, superseded Row, pacing Obj) string { + if ack != nil && ack.S("verified") == "verified" { + if ack.I("accepted") != 0 { + return "acknowledged" + } + return "rejected" + } + if row.S("kind") == MergeTurnGrant && grant != "" { + if grant == mergeturn.GrantAnswered { + return "grant_acknowledged" + } + return "superseded:" + grant + } + state, hold := row.S("state"), row.S("hold_reason") + switch { + case state == Superseded: + return "superseded" + case superseded != nil: + return "superseded:" + superseded.S("reason") + case state == InboxOnly || hold == PushChannelClosed: + return "channel_closed" + case state == Dispatched: + if row.S("kind") == Revision { + return "awaiting_child_receipt" + } + if n := len(attempts); n > 0 && strings.HasPrefix(attempts[n-1].S("recipient_scan"), TurnCheckUndecided+":") { + return "awaiting_ack:" + TurnCheckUndecided + } + return "awaiting_ack" + case state == DeferredBusy: + return "parent_busy" + } + var latest Row + for _, a := range attempts { + if a.S("internal_state") == "settled" { + latest = a + } + } + record := Obj{} + if latest != nil && latest.S("record") != "" { + record = loadsObj(latest.S("record")) + } + failed, _ := get(record, "failedOperation") + failedText, _ := failed.(string) + switch { + case state == HeldUncertain: + if hold != "" { + return "held:" + hold + } + if t, _ := get(record, "turnId"); truthy(t) { + return "turn_accepted" + } + return "outcome_unknown" + case state == WithheldPreSend && latest != nil: + if failedText == "thread/resume" && failure != nil && failure.S("operation") == "settings_check" { + return "settings_rejected" + } + if failedText != "" { + return "withheld:" + failedText + } + return "withheld_pre_send" + case state == WithheldPreSend: + if failure != nil { + return "withheld:" + failure.S("operation") + } + return "withheld_pre_send" + case hold != "": + return "held:" + hold + case state == Sending: + return "in_flight" + case state == Queued && row.S("dispatch_evidence") == HostLostTurn: + return "redelivering:" + HostLostTurn + case state == Queued && str(pacing, "reason") == HourlyCap: + return "awaiting_send:" + HourlyCap + case state == Queued: + return "awaiting_send" + } + return "awaiting_receipt" +} + +// SnapshotItem is one delivery's operator view: the raw state, the reported state and phase. +func (d *Service) SnapshotItem(ctx context.Context, eventID string) (Obj, error) { + row, err := d.Get(ctx, eventID) + if err != nil { + return nil, err + } + attempts, err := all(ctx, d.Store, "SELECT request_id, attempt_no, internal_state, state, affirmative_evidence, operation_observation, recipient_scan, record FROM attempts WHERE event_id = ? ORDER BY attempt_no", eventID) + if err != nil { + return nil, err + } + ack, err := one(ctx, d.Store, "SELECT * FROM acks WHERE event_id = ?", eventID) + if err != nil { + return nil, err + } + failure, err := one(ctx, d.Store, "SELECT * FROM failed_operations WHERE scope_key = ? ORDER BY occurred_at DESC", eventID) + if err != nil { + return nil, err + } + note, err := one(ctx, d.Store, "SELECT reason, noted_at FROM delivery_supersession WHERE event_id = ?", eventID) + if err != nil { + return nil, err + } + grant := "" + if row.S("kind") == MergeTurnGrant { + if grant, err = d.SupersessionReason(ctx, eventID); err != nil { + return nil, err + } + } + var pacing Obj + if (row.S("state") == Queued || row.S("state") == DeferredBusy || row.S("state") == WithheldPreSend) && row.S("hold_reason") == "" { + if pacing, err = d.pacing(ctx, row.S("recipient_task_id"), d.Clock.Now()); err != nil { + return nil, err + } + if reopens, _ := get(pacing, "reopensAt"); pacing != nil && reopens != nil && !row.N("next_eligible_at") && row.F("next_eligible_at") > reopens.(float64) { + pacing = nil + } + } + detail := make([]any, len(attempts)) + for i, a := range attempts { + detail[i] = a + } + var pacingValue any + if pacing != nil { + pacingValue = pacing + } + var noteObj any + if note != nil { + noteObj = Obj{{Key: "reason", Value: note.S("reason")}, {Key: "noted_at", Value: note.S("noted_at")}} + } + return Obj{ + {Key: "eventId", Value: eventID}, {Key: "kind", Value: row.S("kind")}, {Key: "recipient", Value: row.S("recipient_task_id")}, + {Key: "state", Value: row.S("state")}, {Key: "reported", Value: reportedState(row, ack, grant, note)}, + {Key: "attempts", Value: row.I("attempt_count")}, {Key: "holdReason", Value: row.Opt("hold_reason")}, + {Key: "nextEligibleAt", Value: row.Opt("next_eligible_at")}, {Key: "dispatchEvidence", Value: row.Opt("dispatch_evidence")}, + {Key: "acknowledged", Value: ack != nil}, {Key: "attemptDetail", Value: detail}, {Key: "phase", Value: phase(row, attempts, ack, failure, grant, note, pacing)}, {Key: "pacing", Value: pacingValue}, + {Key: "supersededNote", Value: noteObj}, + }, nil +} + +// AttemptMessages is attempt_messages: every attempt's bytes with the status those bytes +// actually reached, read from the attempt and never inferred from the bytes existing. +func (d *Service) AttemptMessages(ctx context.Context, eventID string) ([]any, error) { + rows, err := all(ctx, d.Store, "SELECT a.request_id, a.attempt_no, a.internal_state, a.state, a.record, a.sent_at, m.message FROM attempts a LEFT JOIN attempt_messages m ON m.request_id = a.request_id WHERE a.event_id = ? ORDER BY a.attempt_no", eventID) + if err != nil { + return nil, err + } + out := make([]any, 0, len(rows)) + for _, row := range rows { + var record Obj + if row.S("record") != "" { + record = loadsObj(row.S("record")) + } + status := "uncertain" + switch state := row.S("state"); { + case row.N("message"): + status = "unavailable" + case row.S("internal_state") != "settled": + status = "prepared" + case state == Dispatched || state == Acknowledged: + status = Dispatched + case state == HostLostTurn: + status = HostLostTurn + case record != nil && str(record, "sendAttempted") == "no": + status = "confirmed_unsent" + } + out = append(out, Obj{{Key: "requestId", Value: row.S("request_id")}, {Key: "attemptNo", Value: row.I("attempt_no")}, {Key: "status", Value: status}, + {Key: "deliveryState", Value: row.Opt("state")}, {Key: "sentAt", Value: row.Opt("sent_at")}, {Key: "message", Value: row.Opt("message")}}) + } + return out, nil +} diff --git a/internal/relay/delivery/supersession_test.go b/internal/relay/delivery/supersession_test.go new file mode 100644 index 00000000..5281e89a --- /dev/null +++ b/internal/relay/delivery/supersession_test.go @@ -0,0 +1,292 @@ +package delivery + +import ( + "context" + "database/sql" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// test_supersession.py SUP-1..SUP-8. The daemon's observation pass (todo 29) is replaced by the +// two calls it makes for a finished child turn: resolve_staged, then annotate_predecessors. + +func (f *fixture) advanceTo(number int) { + turn := "turn-dispatch-" + string(rune('0'+number)) + f.host.startTurn(child, turn, "inProgress", "") + mustDo(f.t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := OpenGenerationIn(ctx, f.store, f.clock, f.rid, "dispatch-"+string(rune('0'+number)), "needs_changes_revision", turn) + return err + })) +} + +func (f *fixture) queuedOutcome(outcome string) string { + rid := f.register(regOpts{}) + var payload Obj + if outcome == "ready_for_review" { + payload = f.readyPayload(rid, 1, []string{f.artifact("out.txt", "the deliverable")}, 1, assigned("completed")) + } else { + status := map[string]string{"failed": "failed", "interrupted": "interrupted"}[outcome] + if status == "" { + status = "completed" + } + payload = f.executionPayload(rid, 1, outcome, 1, assigned(status)) + } + _, err := f.accept(payload, store.AcceptOptions{}) + mustDo(f.t, err) + _, err = f.delivery.Enqueue(f.ctx, str(payload, "eventId"), "", "") + mustDo(f.t, err) + return str(payload, "eventId") +} + +func (f *fixture) declare(successor, older string) { + receipt, err := f.delivery.Receipt(f.ctx, older) + mustDo(f.t, err) + _, err = execSQL(f.ctx, f.store, "UPDATE revision_lineage SET supersedes_hash = ? WHERE event_id = ?", str(receipt, "revisionHash"), successor) + mustDo(f.t, err) +} + +func (f *fixture) item(event string) Obj { + it, err := f.delivery.SnapshotItem(f.ctx, event) + mustDo(f.t, err) + out := Obj{} + for _, k := range []string{"state", "reported", "phase", "supersededNote", "holdReason"} { + v, _ := get(it, k) + out = append(out, F{Key: k, Value: v}) + } + return out +} + +func runSUP(t *testing.T, mode string, goSide func(f *fixture, out map[string]any)) { + tree := t.TempDir() + python := runPython(t, tree, "sup", mode) + f := newFixture(t, tree) + out := map[string]any{} + goSide(f, out) + for k, want := range python.Out { + requireSameJSON(t, mode+"."+k, out[k], want) + } + requireSameTables(t, f, python) +} + +func TestSUP01_an_advanced_generation_annotates_older_deliveries_without_rewriting_them(t *testing.T) { + for _, mode := range []string{"queued", "outstanding", "capped", "withheld_cap", "dispatched", "current", "mark"} { + t.Run(mode, func(t *testing.T) { + runSUP(t, mode, func(f *fixture, out map[string]any) { + e := f.queuedOutcome("ready_for_review") + switch mode { + case "outstanding", "current", "mark": + f.host.script = []string{"transport_unknown"} + if mode == "mark" { + f.clock.Advance(3600) + } + f.mustAttempt(e, nil) + case "capped": + _, err := execSQL(f.ctx, f.store, "UPDATE deliveries SET state = ?, hold_reason = ? WHERE event_id = ?", DeferredBusy, "busy_cap", e) + mustDo(t, err) + case "withheld_cap": + _, err := execSQL(f.ctx, f.store, "UPDATE deliveries SET state = ?, hold_reason = ? WHERE event_id = ?", WithheldPreSend, "presend_cap", e) + mustDo(t, err) + case "dispatched": + f.mustAttempt(e, nil) + } + before := f.row(e).S("state") + f.advanceTo(2) + if mode == "current" { + _, err := execSQL(f.ctx, f.store, "DELETE FROM delivery_supersession WHERE event_id = ?", e) + mustDo(t, err) + f.advanceTo(3) + rows, err := all(f.ctx, f.store, "SELECT event_id FROM delivery_supersession") + mustDo(t, err) + if len(rows) != 1 || rows[0].S("event_id") != e { + t.Fatal("only the older generation's outstanding send is annotated") + } + } + if mode == "mark" { + mustDo(t, f.delivery.MarkSuperseded(f.ctx, e, StaleGeneration)) + } + if f.row(e).S("state") != before { + t.Fatal("annotated, never rewritten") + } + out["item"] = f.item(e) + }) + }) + } +} + +func TestSUP02_an_annotated_delivery_reports_as_history(t *testing.T) { + runSUP(t, "dispatched", func(f *fixture, out map[string]any) { + e := f.queuedOutcome("ready_for_review") + f.mustAttempt(e, nil) + f.advanceTo(2) + it := f.item(e) + if str(it, "phase") != "superseded:"+StaleGeneration || str(it, "state") != Dispatched { + t.Fatalf("item %v", it) + } + out["item"] = it + }) +} + +func TestSUP03_a_final_successor_annotates_its_predecessor(t *testing.T) { + for _, mode := range []string{"terminal", "queued_pred", "outstanding_pred"} { + t.Run(mode, func(t *testing.T) { + runSUP(t, mode, func(f *fixture, out map[string]any) { + older := f.queuedOutcome("ready_for_review") + turn := assigned("completed") + if mode == "outstanding_pred" { + f.host.startTurn(child, dispatchTurn, "inProgress", "") + turn = assigned("inProgress") + } + if mode != "queued_pred" { + f.host.script = []string{"transport_unknown"} + f.mustAttempt(older, nil) + } + s := f.readyPayload(f.rid, 1, []string{f.artifact("newer.txt", "the corrected deliverable")}, 2, turn) + _, err := f.accept(s, store.AcceptOptions{}) + mustDo(t, err) + f.declare(str(s, "eventId"), older) + if mode == "outstanding_pred" { + _, err := f.intake.ResolveStaged(f.ctx, store.TurnReference{ThreadID: child, TurnID: dispatchTurn, Status: "completed"}) + mustDo(t, err) + mustDo(t, f.delivery.AnnotatePredecessors(f.ctx, str(s, "eventId"))) + } else { + _, err := f.delivery.Enqueue(f.ctx, str(s, "eventId"), "", "") + mustDo(t, err) + } + it := f.item(older) + note, _ := get(it, "supersededNote") + if note == nil || str(note.(Obj), "reason") != SupersededRevision { + t.Fatalf("item %v", it) + } + out["item"] = it + }) + }) + } +} + +func TestSUP04_a_re_emitted_final_receipt_still_reports_its_stage(t *testing.T) { + tree := t.TempDir() + python := runPython(t, tree, "sup", "reemit") + f := newFixture(t, tree) + rid := f.register(regOpts{}) + payload := f.readyPayload(rid, 1, []string{f.artifact("out.txt", "the deliverable")}, 1, assigned("completed")) + first, err := f.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + again, err := f.accept(payload, store.AcceptOptions{}) + mustDo(t, err) + want := python.Out["again"].(map[string]any) + if first.Stage != "final" || !again.Duplicate || again.Stage != "final" || want["_stage"] != "final" || want["_duplicate"] != true { + t.Fatalf("again %+v", again) + } + requireSameJSON(t, "record", loadsObj(again.Record), withoutUnderscored(want)) + requireSameTables(t, f, python) +} + +func TestSUP05_a_later_execution_only_outcome_survives_a_final_revision_head(t *testing.T) { + runSUP(t, "exec_only", func(f *fixture, out map[string]any) { + reviewable := f.queuedOutcome("ready_for_review") + f.mustAttempt(reviewable, nil) + later := f.executionPayload(f.rid, 1, "failed", 1, assigned("failed")) + _, err := f.accept(later, store.AcceptOptions{}) + mustDo(t, err) + _, err = f.delivery.Enqueue(f.ctx, str(later, "eventId"), "", "") + mustDo(t, err) + var reason string + mustDo(t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + reason, err = f.delivery.SupersessionReason(ctx, str(later, "eventId")) + return err + })) + if reason != "" { + t.Fatalf("suppressed: %s", reason) + } + out["reason"] = nil + out["state"] = f.row(str(later, "eventId")).S("state") + }) +} + +func TestSUP06_enqueueing_clears_the_intent_it_satisfies(t *testing.T) { + for _, mode := range []string{"intent", "stranded"} { + t.Run(mode, func(t *testing.T) { + runSUP(t, mode, func(f *fixture, out map[string]any) { + e := f.queuedOutcome("ready_for_review") + if mode == "intent" { + _, err := execSQL(f.ctx, f.store, "DELETE FROM deliveries WHERE event_id = ?", e) + mustDo(t, err) + } + mustDo(t, f.store.Transaction(f.ctx, func(ctx context.Context, _ *sql.Conn) error { + return f.delivery.RecordIntentIn(ctx, e, f.rid, "completion", parent, "the relationship was paused", f.clock.Now()) + })) + row, err := f.delivery.Enqueue(f.ctx, e, "", "") + mustDo(t, err) + out["row"] = row + if f.one("SELECT 1 AS x FROM delivery_intent WHERE event_id = ?", e) != nil { + t.Fatal("the intent outlived its delivery") + } + }) + }) + } +} + +func TestSUP07_an_older_generation_is_never_sent(t *testing.T) { + for _, outcome := range []string{"ready_for_review", "blocked_needs_input", "failed"} { + t.Run(outcome, func(t *testing.T) { + runSUP(t, "presend_"+outcome, func(f *fixture, out map[string]any) { + e := f.queuedOutcome(outcome) + f.advanceTo(2) + generations := f.count("SELECT COUNT(*) AS c FROM generations") + f.clock.Advance(3600) + record := f.mustAttempt(e, nil) + out["record"] = record + if str(record, "supersededReason") != StaleGeneration || f.row(e).S("hold_reason") != StaleGeneration || len(f.host.sends) != 0 || f.count("SELECT COUNT(*) AS c FROM generations") != generations { + t.Fatalf("record %v", record) + } + }) + }) + } + t.Run("after a busy wait", func(t *testing.T) { + runSUP(t, "busy_release", func(f *fixture, out map[string]any) { + e := f.queuedOutcome("ready_for_review") + f.host.threads[parent].status = "active" + out["first"] = f.mustAttempt(e, nil) + f.advanceTo(2) + f.host.threads[parent].status = "idle" + f.clock.Advance(3600) + out["record"] = f.mustAttempt(e, nil) + if len(f.host.sends) != 0 { + t.Fatal("the wait ending is not permission to send") + } + }) + }) +} + +func TestSUP08_a_newer_final_revision_supersedes_and_a_staged_one_does_not(t *testing.T) { + t.Run("final successor", func(t *testing.T) { + runSUP(t, "newer", func(f *fixture, out map[string]any) { + older := f.queuedOutcome("ready_for_review") + n := f.readyPayload(f.rid, 1, []string{f.artifact("newer.txt", "the corrected deliverable")}, 2, assigned("completed")) + _, err := f.accept(n, store.AcceptOptions{}) + mustDo(t, err) + f.declare(str(n, "eventId"), older) + _, err = f.delivery.Enqueue(f.ctx, str(n, "eventId"), "", "") + mustDo(t, err) + f.clock.Advance(3600) + out["record"] = f.mustAttempt(older, nil) + f.clock.Advance(3600) + out["sent"] = f.mustAttempt(str(n, "eventId"), nil) + }) + }) + t.Run("staged successor", func(t *testing.T) { + runSUP(t, "staged_successor", func(f *fixture, out map[string]any) { + older := f.queuedOutcome("ready_for_review") + st := f.readyPayload(f.rid, 1, []string{f.artifact("staged.txt", "still being written")}, 2, assigned("inProgress")) + _, err := f.accept(st, store.AcceptOptions{}) + mustDo(t, err) + f.clock.Advance(3600) + record := f.mustAttempt(older, nil) + if str(record, "deliveryState") != Dispatched { + t.Fatal("a staged claim is not a replacement") + } + out["record"] = record + }) + }) +} diff --git a/internal/relay/delivery/sync.go b/internal/relay/delivery/sync.go new file mode 100644 index 00000000..fd7018e0 --- /dev/null +++ b/internal/relay/delivery/sync.go @@ -0,0 +1,148 @@ +package delivery + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" + "fmt" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The verdict's sync obligation (sync.py enqueue_verdict_in) and the claim fence the outbox's +// completion rests on (claim, _fenced). The rest of the outbox - next, complete's readback +// validation, fail, retry - is todo 23's. +const ( + SyncNotClaimable = "sync_not_claimable" + coordinationDocument = "coordination_document" + syncLeaseSeconds = 300.0 +) + +func syncCanonical(target, targetRef, kind, rid string, event, generation, revision, verdict, digest, ruling any) string { + render := func(v any) string { + if v == nil { + return "null" + } + return pyStr(v) + } + parts := []string{target, targetRef, kind, rid, render(event), render(generation), render(revision), render(verdict)} + payload := strings.Join(parts, "|") + if digest != nil { + payload += "|criteria=" + render(digest) + } + if ruling != nil { + payload += "|ruling=" + render(ruling) + } + return payload +} + +func renderVerdictSummary(r Relationship, event Row, verdict string, findings []any, record Obj, digest any, ruling int64) string { + lines := []string{ + fmt.Sprintf("%s · %s · %s", r.IssueKey, r.Child.TaskID, verdict), + fmt.Sprintf("generation %d, revision %s", event.I("execution_generation"), event.S("revision_hash")[:12]), + } + if d, ok := digest.(string); ok && d != "" { + line := "criteria set " + d[:12] + if ruling > 1 { + line += fmt.Sprintf(", ruling %d", ruling) + } else if ruling == 1 { + line += ", ruling 1" + } + lines = append(lines, line) + } + if next, _ := get(record, "nextExecutionGeneration"); truthy(next) { + lines = append(lines, fmt.Sprintf("a revision request was queued to the same child under generation %s", pyStr(next))) + } + if len(findings) > 0 { + lines = append(lines, "findings:") + for _, f := range findings { + o := f.(Obj) + line := " " + str(o, "id") + ": " + str(o, "verdict") + if note := str(o, "note"); note != "" { + line += " — " + note + } + lines = append(lines, line) + } + } + return strings.Join(lines, "\n") +} + +// VerdictSync is the SyncHook record_verdict calls inside its transaction. +func VerdictSync(s *store.Store, clock Clock) SyncHook { + return func(ctx context.Context, r Relationship, event Row, verdict string, findings []any, record Obj, digest any, ruling int64) error { + target, err := one(ctx, s, "SELECT * FROM sync_targets WHERE relationship_id = ? AND target = ?", r.ID, coordinationDocument) + if err != nil || target == nil { + return err + } + var identityRuling any + if ruling > 1 { + identityRuling = ruling + } + summaryRuling := ruling + if digest == nil { + summaryRuling = 0 + } + canonical := syncCanonical(coordinationDocument, target.S("target_ref"), "verdict", r.ID, event.S("event_id"), event.I("execution_generation"), event.S("revision_hash"), verdict, digest, identityRuling) + sum := sha256.Sum256([]byte(canonical)) + full := hex.EncodeToString(sum[:]) + id := full[:32] + now := clock.ISO() + inserted, err := execSQL(ctx, s, "INSERT OR IGNORE INTO sync_outbox (sync_id, relationship_id, issue_key, target, target_ref, subject_kind, event_id, execution_generation, revision_hash, verdict, identity_digest, summary, state, attempts, next_attempt_at, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,0,NULL,?,?)", + id, r.ID, r.IssueKey, coordinationDocument, target.S("target_ref"), "verdict", event.S("event_id"), event.I("execution_generation"), event.S("revision_hash"), verdict, full, renderVerdictSummary(r, event, verdict, findings, record, digest, summaryRuling), "pending", now, now) + if err != nil { + return err + } + return journal(ctx, s, "sync_enqueued", id, Obj{{Key: "subjectKind", Value: "verdict"}, {Key: "eventId", Value: event.S("event_id")}, {Key: "criteriaDigest", Value: digest}, {Key: "ruling", Value: identityRuling}, {Key: "inserted", Value: inserted == 1}}, now) + } +} + +// SyncClaim is sync.claim: a lease and a per-claim token, which fences completion. +func SyncClaim(ctx context.Context, s *store.Store, clock Clock, id, owner string, now float64) (Obj, error) { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return nil, err + } + token := hex.EncodeToString(raw) + err := s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + row, err := one(ctx, s, "SELECT * FROM sync_outbox WHERE sync_id = ?", id) + if err != nil { + return err + } + switch { + case row == nil: + return refuse(SyncNotClaimable, "no synchronisation job %s", store.PyRepr(id)) + case row.S("state") == "confirmed": + return refuse(SyncNotClaimable, "%s is already confirmed", store.PyRepr(id)) + case row.S("state") == "failed": + return refuse(SyncNotClaimable, "%s is failed after %d attempts; call retry to resume it deliberately", store.PyRepr(id), row.I("attempts")) + case !row.N("next_attempt_at") && row.F("next_attempt_at") > now: + return refuse(SyncNotClaimable, "%s is backing off until %s", store.PyRepr(id), pyStr(row.Opt("next_attempt_at"))) + case !row.N("lease_until") && row.F("lease_until") > now: + return refuse(SyncNotClaimable, "%s is leased by %s until %s", store.PyRepr(id), pyReprValue(row.Opt("lease_owner")), pyStr(row.Opt("lease_until"))) + } + _, err = execSQL(ctx, s, "UPDATE sync_outbox SET state = ?, lease_owner = ?, lease_until = ?, claim_token = ?, updated_at = ? WHERE sync_id = ?", "claimed", owner, now+syncLeaseSeconds, token, clock.ISO(), id) + return err + }) + if err != nil { + return nil, err + } + return Obj{{Key: "syncId", Value: id}, {Key: "claimToken", Value: token}, {Key: "owner", Value: owner}, {Key: "leaseUntil", Value: now + syncLeaseSeconds}}, nil +} + +// SyncFenced is sync._fenced: only the claim currently held may act on a job. +func SyncFenced(ctx context.Context, s *store.Store, id, token string) (Row, error) { + row, err := one(ctx, s, "SELECT * FROM sync_outbox WHERE sync_id = ?", id) + if err != nil { + return nil, err + } + if row == nil { + return nil, refuse(SyncNotClaimable, "no synchronisation job %s", store.PyRepr(id)) + } + if row.S("claim_token") != token { + return nil, refuse(SyncNotClaimable, "this claim token is not the one currently held for %s", store.PyRepr(id)) + } + return row, nil +} diff --git a/internal/relay/delivery/testdata/capture.py b/internal/relay/delivery/testdata/capture.py new file mode 100644 index 00000000..3d587291 --- /dev/null +++ b/internal/relay/delivery/testdata/capture.py @@ -0,0 +1,100 @@ +"""Runs every test of one Python test module, each over its own tree the Go tests share, and +records every value each test asserts. + +argv: . Each test runs with tempfile.mkdtemp returning /., so +paths, revision hashes, event and request ids are the ones the Go test derives in the same tree. +Every assertion still runs; each also appends the value the Python code produced (the first +argument of assertEqual, the expression of assertTrue, ...) to the test's capture list. The Go +mirror of the test performs the same steps and must produce the same list, so every asserted +value is compared with what Python computed rather than with a constant. /capture.json +holds {captures, problems, tables, sends}. +""" +import json +import os +import shutil +import sqlite3 +import sys +import tempfile +import unittest + +ROOT, MODULE = sys.argv[1:3] +_rmtree = shutil.rmtree + + +def _keep_trees(path, *args, **kwargs): + if os.path.abspath(path).startswith(os.path.abspath(ROOT)): + return None + return _rmtree(path, *args, **kwargs) + + +shutil.rmtree = _keep_trees +captures = [] + + +def plain(value): + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {str(k): plain(v) for k, v in value.items()} + if isinstance(value, (set, frozenset)): + return sorted((plain(v) for v in value), key=lambda v: json.dumps(v, sort_keys=True)) + if isinstance(value, (list, tuple)): + return [plain(v) for v in value] + try: + return {k: plain(value[k]) for k in value.keys()} + except Exception: # noqa: BLE001 + return repr(value) + + +def recorder(name, pick): + original = getattr(unittest.TestCase, name) + + def method(self, *args, **kwargs): + captures.append(plain(pick(*args))) + return original(self, *args, **kwargs) + return method + + +for _name, _pick in { + "assertEqual": lambda a, b, *r: a, "assertNotEqual": lambda a, b, *r: a, + "assertTrue": lambda a, *r: bool(a), "assertFalse": lambda a, *r: bool(a), + "assertIsNone": lambda a, *r: a, "assertIsNotNone": lambda a, *r: a is not None, + "assertIn": lambda a, b, *r: a in b, "assertNotIn": lambda a, b, *r: a in b, + "assertLessEqual": lambda a, b, *r: a <= b, "assertLess": lambda a, b, *r: a < b, +}.items(): + setattr(unittest.TestCase, _name, recorder(_name, _pick)) + + +def flatten(suite): + for item in suite: + if isinstance(item, unittest.TestSuite): + yield from flatten(item) + else: + yield item + + +suite = unittest.defaultTestLoader.loadTestsFromName(f"tests.{MODULE}") +for case in flatten(suite): + name = f"{type(case).__name__}.{case._testMethodName}" + tree = os.path.join(ROOT, name) + os.makedirs(tree) + tempfile.mkdtemp = lambda prefix=None, tree=tree: tree + captures.clear() + result = unittest.TestResult() + case.run(result) + problems = [tb for _t, tb in result.failures + result.errors] + tables = {} + path = os.path.join(tree, "state", "relay.sqlite3") + if os.path.exists(path): + db = sqlite3.connect(path) + db.row_factory = sqlite3.Row + for (table,) in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ('schema_meta','sqlite_sequence') ORDER BY name"): + rows = [dict(r) for r in db.execute(f"SELECT * FROM {table} ORDER BY rowid")] + if rows: + tables[table] = rows + db.close() + adapter = getattr(case, "adapter", None) + sends = [list(s) for s in adapter.sends] if adapter is not None else [] + with open(os.path.join(tree, "capture.json"), "w") as handle: + json.dump({"captures": list(captures), "problems": problems, "tables": tables, + "sends": sends}, handle, default=repr) diff --git a/internal/relay/delivery/testdata/cliseed.py b/internal/relay/delivery/testdata/cliseed.py new file mode 100644 index 00000000..ff1931fb --- /dev/null +++ b/internal/relay/delivery/testdata/cliseed.py @@ -0,0 +1,12 @@ +"""Seed a relationship for the CLI parity test, through the real Python registry.""" +import os, sys +from codex_session_relay.clock import FakeClock +from codex_session_relay.models import Endpoint +from codex_session_relay.registry import Registry, record_settings +from codex_session_relay.store import Store +state, root = sys.argv[1], sys.argv[2] +os.makedirs(root, exist_ok=True) +store = Store(os.path.join(state, "relay.sqlite3")) +clock = FakeClock() +rel = Registry(store, clock).register(parent=Endpoint("01parent-task", "host-a", cwd="/parent"), child=Endpoint("01child-task", "host-a", cwd=root), issue_key="REL-1", artifact_roots=[root], allowed_recipients=["01parent-task", "01child-task"], dispatch_request_id="dispatch-1", dispatch_turn_id="turn-dispatch-1") +print(rel["relationshipId"]) diff --git a/internal/relay/delivery/testdata/drl_answers.json b/internal/relay/delivery/testdata/drl_answers.json new file mode 100644 index 00000000..cfd34ab3 --- /dev/null +++ b/internal/relay/delivery/testdata/drl_answers.json @@ -0,0 +1,14 @@ +{ + "agree_completion": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [], "levels": [{"scopeKind": "issue", "scopeKey": "ISS-1", "owner": {"taskId": "child-task", "revision": 1}, "depth": 0}, {"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}}, + "agree_revision": {"kind": "revision_request", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [], "levels": [{"scopeKind": "issue", "scopeKey": "ISS-1", "owner": {"taskId": "child-task", "revision": 2}, "depth": 0}, {"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}}, + "owner_changed": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "replacement-parent", "revision": 4}, "depth": 1}]}}, + "drift_resolved": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [{"contention": "owner_drift", "scopeKind": "project", "scopeKey": "PRJ-1"}], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}}, + "live_beside_audit": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [{"scopeKey": "PRJ-1", "reason": "duplicate_scope_owner", "incumbent": "a", "challenger": "b", "at": "2026-01-01T00:00:00Z", "detail": null}, {"contention": "owner_drift", "scopeKind": "project", "scopeKey": "PRJ-1"}], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}}, + "unreadable": {"kind": "completion_event", "answer": {"state": "unreadable", "readable": false, "levels": [], "gaps": [], "contention": []}}, + "nothing_found": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "contention": [], "gaps": [{"gap": "project_without_parent", "scopeKind": "project", "scopeKey": "PRJ-1"}], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": null, "depth": 1}]}}, + "unregistered": {"kind": "completion_event", "answer": {"state": "unregistered", "readable": true, "levels": [], "contention": [], "gaps": [{"gap": "unscoped_assignment", "relationshipId": "rel-1"}]}}, + "two_owners": {"kind": "completion_event", "answer": {"state": "ambiguous", "readable": true, "levels": [], "gaps": [], "contention": [{"contention": "competing_owners", "candidates": ["parent-task", "other-parent"]}]}}, + "instruction_conflict": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [{"contention": "instruction_conflict", "scopeKind": "project", "scopeKey": "PRJ-1", "digest": "d0"}], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}}, + "undefined_direction": {"kind": "gossip", "answer": {"state": "registered", "readable": true, "levels": [], "gaps": [], "contention": []}}, + "audit_only": {"kind": "completion_event", "answer": {"state": "registered", "readable": true, "gaps": [], "contention": [{"at": "2026-01-01T00:00:00Z", "scopeKind": "project", "scopeKey": "PRJ-1", "reason": "duplicate_scope_owner", "incumbent": "parent-task", "challenger": "someone-else", "detail": null}], "levels": [{"scopeKind": "project", "scopeKey": "PRJ-1", "owner": {"taskId": "parent-task", "revision": 3}, "depth": 1}]}} +} diff --git a/internal/relay/delivery/testdata/grantstage.py b/internal/relay/delivery/testdata/grantstage.py new file mode 100644 index 00000000..c71239ed --- /dev/null +++ b/internal/relay/delivery/testdata/grantstage.py @@ -0,0 +1,58 @@ +"""Stages the merge-turn half of test_unknown_send_lost's AnUncertainGrantIsReadAsStatusReadsIt in +Python, over a store the Go test then reads and writes itself. + +argv: . The merge turn's request, promotion, grant notice, acknowledgement and regrant are +mergeturn.py's (todo 26 ports them); reconciliation and the status reading, which the property is +about, run in Go on the same file. The first line printed is the staged state; after it the script +reads one command per line (advance | answer | regrant | quit) and answers each with a line. +""" +import json +import os +import sys +import tempfile + +TREE = sys.argv[1] +tempfile.mkdtemp = lambda prefix=None: TREE + +from tests.support import PARENT # noqa: E402 +from tests.test_merge_turn_wake import MergeTurnWakeTestCase # noqa: E402 + + +class Case(MergeTurnWakeTestCase): + def runTest(self): + pass + + +c = Case() +c.setUp() +# promoted(), keeping the state of the claim it queued (the value the Python test asserts). +from tests.test_merge_turn_wake import PROJECT_A, PROJECT_B # noqa: E402 +held = c.claim(c.beta, PROJECT_B, "head-b") +waiter = c.claim(c.alpha, PROJECT_A, "head-a") +waiting = waiter["state"] +c.turns.release(held["turnId"], actor=c.beta.task_id, disposition="returned", + reason="not ready after all") +turn = waiter["turnId"] +event = c.wakes()[0]["event_id"] +c.adapter.script("transport_unknown") +record = c.attempt(event) +c.store.db.commit() if c.store.db.in_transaction else None +print(json.dumps({ + "waiting": waiting, "turn": turn, "event": event, "record": record, + "receipt": c.adapter.ledger[record["requestId"]], + "sends": [list(s) for s in c.adapter.sends], "now": c.clock.now(), + "threads": sorted(c.adapter.threads), "store": str(c.store.path), +}), flush=True) +for line in sys.stdin: + words = line.split() + if not words or words[0] == "quit": + break + if words[0] == "advance": + c.clock.advance(float(words[1])) + elif words[0] == "answer": + c.answer_grant(turn, PARENT) + elif words[0] == "regrant": + c.turns.declare_ready(turn, actor=PARENT, ready=True, candidate_head="head-a2") + c.store.db.commit() if c.store.db.in_transaction else None + print(json.dumps({"ok": words[0]}), flush=True) +c.store.close() diff --git a/internal/relay/delivery/testdata/markerops.py b/internal/relay/delivery/testdata/markerops.py new file mode 100644 index 00000000..34d7ab72 --- /dev/null +++ b/internal/relay/delivery/testdata/markerops.py @@ -0,0 +1,215 @@ +"""Drives the real Python marker and intent modules through a JSON list of operations. + +argv: . stdin: {"ops": [...], "env": {...}}. Every operation's answer is printed in order as +one JSON list: {"ok": value} for a return, {"reason", "detail"} for a RelayError, {"error": ...} +for any other exception. The Go test runs the same list through its own port over the same tree +and compares the whole list. Paths are relative to the tree: markers/ is the marker root, work/ +the workspace, state/relay.sqlite3 the relay store. +""" +import json +import os +import sqlite3 +import sys +from pathlib import Path + +TREE = Path(sys.argv[1]) +spec = json.load(sys.stdin) +for key, value in (spec.get("env") or {}).items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value.replace("", str(TREE)) + +from codex_session_relay import intent, marker # noqa: E402 +from codex_session_relay.errors import RelayError # noqa: E402 +from codex_session_relay.store import Store # noqa: E402 + +ROOT = TREE / "markers" +WORK = TREE / "work" +WORK.mkdir(parents=True, exist_ok=True) +DB = TREE / "state" / "relay.sqlite3" +store = None +held = None + + +def the_store(): + global store + if store is None: + store = Store(str(DB)) + return store + + +def path(value): + return str(value).replace("", str(TREE)) if isinstance(value, str) else value + + +def assignment(op): + if "assignment" in op: + return op["assignment"] + return marker.assignment_id(op.get("dispatch", "dispatch-request-1")) + + +def adir(op): + return marker.assignment_dir(ROOT, WORK, marker.assignment_id(op.get("dispatch", "dispatch-request-1"))) + + +def facts(op): + found, unreadable = marker.read_assignment(adir(op)) + return found, unreadable + + +def run(op): + global held + kind = op["op"] + if kind == "declare": + kw = {k: op[k] for k in ("criteria_source", "baseline_revision", "authorized_settings") if k in op} + if "db_path" in op: + kw["db_path"] = path(op["db_path"]) + return intent.declare_intent(ROOT, workspace=Path(path(op.get("workspace", str(WORK)))), + dispatch_request_id=op.get("dispatch", "dispatch-request-1"), + issue_key=op.get("issue_key", "REL-1"), + declared_at=op.get("declared_at", "2026-01-01T00:00:00+00:00"), **kw) + if kind == "attempt": + return intent.record_attempt(ROOT, workspace=WORK, assignment=assignment(op), outcome=op["outcome"], + task_id=op.get("task_id"), at=op.get("at", "2026-01-01T00:00:00+00:00")) + if kind == "claim": + return intent.publish_claim(ROOT, workspace=WORK, assignment=assignment(op), session_id=op["session"], + dispatch_request_id=op.get("claim_dispatch", "dispatch-request-1"), + first_turn_id="turn-1", at=op.get("at", "2026-01-01T00:00:00+00:00")) + if kind == "bind": + return intent.bind(ROOT, workspace=WORK, assignment=assignment(op), session_id=op["session"], + task_id=op["task"], at=op.get("at", "2026-01-01T00:00:00+00:00")) + if kind == "open_generation": + s = the_store() + rid, generation, current = op["relationship_id"], op.get("generation", 1), op.get("current") + s.db.execute( + "INSERT OR IGNORE INTO relationships (relationship_id, issue_key, status, parent_task_id," + " parent_host_id, child_task_id, child_host_id, execution_generation, artifact_roots," + " allowed_recipients, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)", + (rid, "REL-1", "active", "01parent-task", "host-a", "01child-task", "host-a", + generation if current is None else current, "[]", "[]", "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00")) + if current is not None: + s.db.execute("UPDATE relationships SET execution_generation = ? WHERE relationship_id = ?", + (current, rid)) + s.db.execute( + "INSERT OR IGNORE INTO generations (relationship_id, execution_generation, dispatch_request_id," + " anchor_state, opened_at) VALUES (?,?,?,?,?)", + (rid, generation, op.get("generation_dispatch", "dispatch-request-1"), "bound", + "2026-01-01T00:00:00+00:00")) + return None + if kind == "register": + db_path = path(op.get("db_path", "/state/relay.sqlite3")) + return intent.register_relationship(ROOT, workspace=WORK, assignment=assignment(op), + relationship_id=op["relationship_id"], + dispatch_request_id=op.get("register_dispatch", "dispatch-request-1"), + at="2026-01-01T00:00:00+00:00", db_path=db_path) + if kind == "hold_begin": + the_store().db.execute("BEGIN IMMEDIATE") + return None + if kind == "hold_end": + the_store().db.execute("ROLLBACK") + return None + if kind == "relay_tables": + s = the_store() + return {"relationships": [dict(r) for r in s.all("SELECT * FROM relationships")], + "generations": [dict(r) for r in s.all("SELECT * FROM generations")], + "journal": s.one("SELECT COUNT(*) AS n FROM journal")["n"]} + if kind == "resolution": + found, _ = facts(op) + entries = [] + for fact_id in op.get("facts", []): + fact = next(f for f in (found.get("attempts", []) + found.get("claims", []) + + found.get("conflicts", [])) if f["factId"] == fact_id) + digest = "0" * 64 if op.get("digest") == "zero" else marker.fact_digest(fact) + entries.append({"factId": fact_id, "digest": digest}) + return intent.publish_resolution(ROOT, workspace=WORK, assignment=assignment(op), + chosen_task_id=op.get("task"), chosen_session_id=op.get("session"), + reason=op.get("reason", "r"), at="2026-01-01T00:05:00+00:00", + adjudicated=entries) + if kind == "publish": + target = adir(op) / op["path"] if "path" in op else Path(path(op["target"])) + return marker.publish(target, op["payload"], root=ROOT if op.get("confined") else None) + if kind == "write_raw": + target = adir(op) / op["path"] if "path" in op else Path(path(op["target"])) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(op["text"], encoding="utf-8") + return None + if kind == "unlink": + (adir(op) / op["path"]).unlink() + return None + if kind == "exists": + return (TREE / op["path"]).exists() if "tree_path" not in op else (TREE / op["tree_path"]).exists() + if kind == "exists_in": + return (adir(op) / op["path"]).exists() + if kind == "listdir": + target = Path(path(op["target"])) + return sorted(p.name for p in target.iterdir()) + if kind == "read_file": + return json.loads(Path(path(op["target"])).read_text(encoding="utf-8")) + if kind == "facts": + found, unreadable = facts(op) + return {"facts": found, "unreadable": unreadable} + if kind == "state": + return intent.derive_assignment_state(facts(op)[0], op.get("now", "2026-01-01T00:05:00+00:00")) + if kind == "malformed": + return intent.malformed(facts(op)[0]) + if kind == "counters": + return intent.malformed_counters(op["value"]) + if kind == "contested": + return intent.identity_contested(facts(op)[0]) + if kind == "covered": + found = facts(op)[0] + fact = next(f for f in found.get("claims", []) if f["factId"] == op["fact"]) + return intent.covered(fact, intent._resolutions(found)) + if kind == "claimant": + found = facts(op)[0] + return intent.claimant(next(f for f in found.get("claims", []) if f["factId"] == op["fact"])) + if kind == "correlated": + return intent.correlated(facts(op)[0], op["session"]) + if kind == "select": + workspace = Path(path(op.get("workspace", str(WORK)))) + found, got, unreadable = intent.select_assignment(ROOT, workspace, op["session"]) + return {"assignment": found.name if found is not None else None, + "facts": got, "unreadable": unreadable} + if kind == "disposition": + return intent.publish_disposition(ROOT, workspace=WORK, assignment=assignment(op), + session_id=op["session"], turn_id=op["turn"], + outcome=op["outcome"], at="2026-01-01T00:00:00+00:00") + if kind == "read_disposition": + found, readable = marker.read_disposition(adir(op), op["session"], op["turn"]) + return {"found": found, "readable": readable} + if kind == "digest": + return marker.fact_digest(op["payload"]) + if kind == "named": + return [marker.named(v) for v in op["values"]] + if kind == "same": + return [marker.same_identity(a, b) for a, b in op["pairs"]] + if kind == "workspace_key": + return marker.workspace_key(Path(path(op["workspace"]))) + if kind == "symlink": + os.symlink(path(op["to"]), path(op["link"])) + return None + if kind == "marker_root": + chosen = marker.resolve_marker_root(path(op.get("explicit"))) + return chosen.to_record() + if kind == "set_env": + if op["value"] is None: + os.environ.pop(op["name"], None) + else: + os.environ[op["name"]] = path(op["value"]) + return None + raise ValueError("unknown op " + kind) + + +out = [] +for op in spec["ops"]: + try: + out.append({"ok": run(op)}) + except RelayError as error: + out.append({"reason": error.reason.value if error.reason else None, "detail": error.detail}) + except Exception as error: # noqa: BLE001 + out.append({"error": type(error).__name__ + ": " + str(error)}) +if store is not None: + store.close() +print(json.dumps(out)) diff --git a/internal/relay/delivery/testdata/ordadapter.py b/internal/relay/delivery/testdata/ordadapter.py new file mode 100644 index 00000000..f4aa74ec --- /dev/null +++ b/internal/relay/delivery/testdata/ordadapter.py @@ -0,0 +1,54 @@ +"""ORD-6..ORD-9 through the real Python adapter and supervisor channel. argv: .""" +import json, sys, tempfile +TREE, MODE = sys.argv[1], sys.argv[2] +tempfile.mkdtemp = lambda prefix=None: TREE +from tests import test_bridge_adapter as seam +from tests.test_on_request_delivery import record_based, RealAdapterRoutes, OnRequestSupervisor +from codex_session_relay.transport import classify_operation_receipt +from codex_session_relay import supervisorchannel as channel_module +out = {} +def keep(receipt): + return {k: receipt.get(k) for k in ("status", "rpcError", "settingsNotes", "settingsFindings", "error", "statusBeforeResume")} +if MODE in ("transmitted", "settings_free", "untrusted"): + case = RealAdapterRoutes("test_untrusted_stays_stored_not_woken"); case.setUp() + policy = "untrusted" if MODE == "untrusted" else "on-request" + adapter, calls = case._adapter(resume=seam.authorized_resume_response(approvalPolicy=policy)) + settings = record_based() if MODE == "settings_free" else seam.AUTHORIZED + receipt = adapter.send_message("del-a1c000000000-" + MODE[:2], "thread-1", "hi", settings) + out["receipt"] = keep(receipt) + out["methods"] = [m for m, _ in calls] + out["resumes"] = [p for m, p in calls if m == "thread/resume"] + out["delivery_state"] = classify_operation_receipt(receipt).delivery_state + out["settings"] = settings.data + out["settingsFree"] = settings.settings_free_resume + out["resumed"] = seam.authorized_resume_response(approvalPolicy=policy) + case.doCleanups() +else: + case = OnRequestSupervisor("test_an_on_request_supervisor_receives_the_push"); case.setUp() + from tests.test_supervisor_channel import SUPERVISOR + if MODE == "sup_untrusted": + case.adapter.threads[SUPERVISOR].approval_policy = "untrusted" + _one, mid = case.staged() + out["record"] = case.channel.attempt(mid, case.adapter) + out["state"] = case.channel.get(mid)["state"] + elif MODE == "sup_on_request": + case.adapter.threads[SUPERVISOR].approval_policy = "on-request" + _one, mid = case.staged() + out["record"] = case.channel.attempt(mid, case.adapter) + out["state"] = case.channel.get(mid)["state"] + else: + case.adapter.threads[SUPERVISOR].approval_policy = "on-request" + existing = case.adapter.start_turn(SUPERVISOR, status="inProgress") + out["turnStartedAt"] = existing.started_at + case.clock.advance(600) + _one, mid = case.staged() + case.adapter.script("steer_existing") + out["record"] = case.channel.attempt(mid, case.adapter) + out["sentAt"] = case.store.one("SELECT transport_started_at FROM supervisor_attempts WHERE message_id = ? ORDER BY attempt_no DESC LIMIT 1", (mid,))["transport_started_at"] + out["verified"] = case.read_back(mid, existing.turn_id)["verified"] + out["state"] = case.channel.get(mid)["state"] + before = len(case.adapter.sends); case.clock.advance(100000) + case.channel.attempt(mid, case.adapter, now=case.clock.now()) + out["resent"] = len(case.adapter.sends) - before + case.doCleanups() +print(json.dumps(out, default=repr)) diff --git a/internal/relay/delivery/testdata/pyscenario.py b/internal/relay/delivery/testdata/pyscenario.py new file mode 100644 index 00000000..0fa64d45 --- /dev/null +++ b/internal/relay/delivery/testdata/pyscenario.py @@ -0,0 +1,56 @@ +"""Drives the real Python delivery package over a fixture tree Go shares, then dumps the store. + +argv: . The tree is the Go test's t.TempDir(): Python's RelayTestCase is pointed +at it, so artifact paths, revision hashes and event ids are the ones the Go run derives. +""" +import json +import os +import sqlite3 +import sys +import tempfile + +TREE, NAME = sys.argv[1], sys.argv[2] +tempfile.mkdtemp = lambda prefix=None: TREE + +from tests.support import CHILD, PARENT, DeliveryTestCase # noqa: E402 +from codex_session_relay.errors import RelayError # noqa: E402 + + +class Case(DeliveryTestCase): + def runTest(self): + pass + + +def refusal(call, *args, **kwargs): + try: + return {"ok": call(*args, **kwargs)} + except RelayError as error: + return {"reason": error.reason.value if error.reason else None, "detail": error.detail} + + +def plain(value): + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, dict): + return {k: plain(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [plain(v) for v in value] + try: + return {k: plain(value[k]) for k in value.keys()} + except Exception: # noqa: BLE001 + return repr(value) + + +c = Case() +c.setUp() +out = {} +exec(open(os.path.join(os.path.dirname(__file__), "scenarios", NAME + ".py")).read()) +c.store.db.commit() if c.store.db.in_transaction else None +db = sqlite3.connect(os.path.join(TREE, "state", "relay.sqlite3")) +db.row_factory = sqlite3.Row +tables = {} +for (name,) in db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name NOT IN ('schema_meta','sqlite_sequence') ORDER BY name"): + rows = [dict(r) for r in db.execute(f"SELECT * FROM {name} ORDER BY rowid")] + if rows: + tables[name] = rows +print(json.dumps({"out": plain(out), "tables": tables, "sends": [list(s) for s in c.adapter.sends]}, default=repr)) diff --git a/internal/relay/delivery/testdata/scenarios/_vcu.py b/internal/relay/delivery/testdata/scenarios/_vcu.py new file mode 100644 index 00000000..d37cdc0b --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/_vcu.py @@ -0,0 +1,25 @@ +from codex_session_relay import identity +from codex_session_relay.criteria import CriteriaService, set_digest +from codex_session_relay.currency import head_revision +SET = [{"id": "c1", "title": "the endpoint returns the agreed shape"}, {"id": "c2", "title": "a malformed request is refused"}] +def acknowledged(recipients=None, turn_id="ack-turn", text="the deliverable"): + _r, e = c.queued_event(recipients=recipients or [PARENT, CHILD], text=text) + c.attempt(e); c.clock.advance(5) + t = c.adapter.start_turn(PARENT, turn_id=turn_id, status="inProgress") + c.ack.acknowledge(e, ack_turn_id=t.turn_id, ack_proof=identity.ack_proof(e, t.turn_id), accepted=True, adapter=c.adapter) + return e +def advance(): + return c.registry.open_generation(c._rid, dispatch_request_id="newer-execution", reason="needs_changes_revision", dispatch_turn_id="newer-turn") +def second(text="a different revision"): + return c.ready_event(register=False, text=text)[1] +def verdict(e, v, turn="v1", **kw): + return refusal(c.ack.record_verdict, e, verdict=v, verdict_turn_id=turn, **kw) +def supersede(e, text, predecessor): + p = c.ready_payload(c.registry.get(c._rid), [c.artifact("out.txt", text)]) + c.intake.accept_child_receipt(p, observation=c.assigned_turn(), supersedes_revision=predecessor) + return p +crit = CriteriaService(c.store, c.clock) +def register_criteria(entries=None): + return crit.register(c._rid, entries or SET, source_ref="https://linear.app/doc/1") +def head(g=1): + return head_revision(c.store.db, c._rid, g) diff --git a/internal/relay/delivery/testdata/scenarios/anb.py b/internal/relay/delivery/testdata/scenarios/anb.py new file mode 100644 index 00000000..ca3044de --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/anb.py @@ -0,0 +1,78 @@ +from unittest import mock +from codex_session_relay import identity +from codex_session_relay.delivery import REVISION +from codex_session_relay.models import TurnRef +def revision_pending(): + _r, e = c.queued_event(recipients=[PARENT, CHILD]) + c.attempt(e); c.clock.advance(5) + t = c.adapter.start_turn(PARENT, turn_id="ack-turn", status="inProgress") + c.ack.acknowledge(e, ack_turn_id=t.turn_id, ack_proof=identity.ack_proof(e, t.turn_id), accepted=True, adapter=c.adapter) + c.ack.record_verdict(e, verdict="needs_changes", verdict_turn_id="verdict-1") + return c.store.one("SELECT * FROM deliveries WHERE kind = ?", (REVISION,))["event_id"] +def gen2(): + return c.registry.generation(c._rid, 2) +def child_receipt(): + rel = c.registry.get(c._rid) + anchor = gen2()["dispatchTurnId"] + turn = c.assigned_turn(thread=CHILD, turn=anchor or "turn-unbound") + return refusal(c.accept, c.ready_payload(rel, [c.artifact("fixed.txt", "the correction")], generation=2, turn=turn)) +def lost_settle_write(): + rev = revision_pending(); c.clock.advance(3600) + c.adapter.script("in_progress") + rec = c.delivery.attempt(rev, c.adapter, now=c.clock.now()) + turn = c.adapter.start_turn(CHILD, status="inProgress") + c.adapter.ledger[rec["requestId"]] = {"requestId": rec["requestId"], "status": "accepted", "resumed": {"approvalPolicy": "never"}, "turnId": turn.turn_id} + return rev, turn.turn_id +m = sys.argv[3] +if m in ("tick", "recovery", "unbound", "idempotent", "never", "retired_ready", "retired_failed", "retired_daemon"): + rev = revision_pending(); c.clock.advance(3600) + if m == "never": c.adapter.script("busy") + out["record"] = c.delivery.attempt(rev, c.adapter, now=c.clock.now()) + out["pending"] = gen2()["anchorState"] + if m == "unbound": + out["receipt"] = child_receipt() + elif m == "tick": + # The tick's two binding passes around its reconciliation (daemon.tick, todo 29). + out["bound"] = [c.ack.bind_pending_anchors(), c.reconciler.recover_on_start(c.adapter)["reconciled"], c.ack.bind_pending_anchors()] + out["gen2"] = gen2(); out["receipt"] = child_receipt() + elif m in ("recovery", "idempotent", "never"): + out["bound"] = c.ack.bind_pending_anchors() + out["again"] = c.ack.bind_pending_anchors() + out["gen2"] = gen2() + if m == "recovery": out["receipt"] = child_receipt() + else: + c.ack.bind_pending_anchors() + if m == "retired_ready": + out["receipt"] = child_receipt() + elif m == "retired_failed": + rel = c.registry.get(c._rid) + turn = c.assigned_turn(thread=CHILD, turn=gen2()["dispatchTurnId"]) + out["receipt"] = refusal(c.accept, c.execution_payload(rel, "failed", generation=2, turn=turn)) + else: + anchor = gen2()["dispatchTurnId"] + c.adapter.start_turn(CHILD, turn_id=anchor, status="failed") + out["receipt"] = c.intake.daemon_observation(c._rid, TurnRef(CHILD, anchor, "failed")) + with c.store.transaction() as db: + out["reason"] = c.delivery._supersession_reason(db, rev) +elif m == "reconcile": + rev = revision_pending(); c.clock.advance(3600) + c.adapter.script("transport_unknown") + out["record"] = c.delivery.attempt(rev, c.adapter, now=c.clock.now()) + out["promoted"] = c.reconciler.recover_on_start(c.adapter) + out["bound"] = c.ack.bind_pending_anchors() + out["gen2"] = gen2(); out["state"] = c.delivery.get(rev)["state"] +elif m == "promotion": + rev, turn = lost_settle_write() + out["recovered"] = c.reconciler.recover_on_start(c.adapter) + out["gen2"] = gen2(); out["receipt"] = child_receipt() +elif m == "conflict": + rev, turn = lost_settle_write() + c.registry.bind_anchor(c._rid, 2, dispatch_turn_id="a-different-turn", source="dispatch_receipt") + out["recovered"] = c.reconciler.recover_on_start(c.adapter) + out["gen2"] = gen2() +elif m == "stale": + rev, turn = lost_settle_write() + c.store.db.execute("UPDATE deliveries SET attempt_count = attempt_count + 1 WHERE event_id = ?", (rev,)) + with mock.patch.object(c.reconciler, "_is_current", return_value=True): + out["recovered"] = c.reconciler.recover_on_start(c.adapter) + out["gen2"] = gen2() diff --git a/internal/relay/delivery/testdata/scenarios/crr.py b/internal/relay/delivery/testdata/scenarios/crr.py new file mode 100644 index 00000000..ec34d72c --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/crr.py @@ -0,0 +1,40 @@ +from codex_session_relay.criteria import CriteriaService +crit = CriteriaService(c.store, c.clock) +SET = [{"id": " c2 ", "title": " malformed requests are refused ", "required": 0}, {"id": "c1", "title": "the endpoint returns the agreed shape"}] +SOURCE = "https://linear.app/doc/1" +m = sys.argv[3] +if m == "refused": + out["r"] = [refusal(crit.ensure_registered, "rel-1", e, source_ref=SOURCE) for e in ([], [{"id": "c1", "title": " "}], [{"id": "c1", "title": "one"}, "not-an-object"], [{"id": "c1", "title": "one"}, {"id": " c1 ", "title": "again"}])] + out["mode"] = crit.mode("rel-1") +elif m == "first": + out["r"] = crit.ensure_registered("rel-1", list(reversed(SET)), source_ref=SOURCE) +elif m == "replay": + crit.ensure_registered("rel-1", SET, source_ref=SOURCE) + c.clock.advance(30) + out["r"] = crit.ensure_registered("rel-1", [{"id": "c2", "title": "malformed requests are refused", "required": False}, {"id": "c1", "title": " the endpoint returns the agreed shape "}], source_ref=SOURCE) +elif m == "changed": + crit.ensure_registered("rel-1", SET, source_ref=SOURCE) + out["r"] = [refusal(crit.ensure_registered, "rel-1", e, source_ref=s) for e, s in ( + ([{"id": "c1", "title": "the endpoint returns the agreed shape"}, {"id": "c2", "title": "a different refusal", "required": False}], SOURCE), + (SET, "https://linear.app/doc/2"), + ([{"id": "c2", "title": "malformed requests are refused", "required": True}, {"id": "c1", "title": "the endpoint returns the agreed shape"}], SOURCE), + (SET + [{"id": "c3", "title": "one more obligation"}], SOURCE))] +elif m == "replace": + crit.ensure_registered("rel-1", SET, source_ref=SOURCE) + c.clock.advance(5) + out["r"] = crit.register("rel-1", [{"id": "c9", "title": "the replacement obligation", "required": False}], source_ref="operator") + out["r2"] = refusal(crit.ensure_registered, "rel-1", SET, source_ref=SOURCE) +else: + from codex_session_relay.criteria import set_digest + N = [{"id": "c1", "title": "the endpoint returns the agreed shape", "required": True}, {"id": "c2", "title": "malformed requests are refused", "required": False}] + d, now = set_digest(N), c.clock.iso() + with c.store.transaction() as db: + if m == "corrupt": + db.execute("INSERT INTO canonical_criteria VALUES (?,?,?,?,?,?,?)", ("rel-1", "c1", N[0]["title"], 1, SOURCE, d, now)) + db.execute("INSERT INTO canonical_criteria VALUES (?,?,?,?,?,?,?)", ("rel-1", "c2", N[1]["title"], 0, "other-source", "not-the-digest", now)) + db.execute("INSERT INTO verification_mode VALUES (?,?,?)", ("rel-1", "legacy", now)) + else: + db.execute("INSERT INTO canonical_criteria VALUES (?,?,?,?,?,?,?)", ("rel-1", "c1", N[0]["title"], 2, SOURCE, d, now)) + db.execute("INSERT INTO canonical_criteria VALUES (?,?,?,?,?,?,?)", ("rel-1", "c2", N[1]["title"], 0, SOURCE, d, now)) + db.execute("INSERT INTO verification_mode VALUES (?,?,?)", ("rel-1", "managed", now)) + out["r"] = refusal(crit.ensure_registered, "rel-1", SET, source_ref=SOURCE) diff --git a/internal/relay/delivery/testdata/scenarios/del01.py b/internal/relay/delivery/testdata/scenarios/del01.py new file mode 100644 index 00000000..876ab0b5 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del01.py @@ -0,0 +1,4 @@ +_r, e = c.queued_event() +out["first"] = dict(c.delivery_row(e)) +out["again"] = dict(c.delivery.enqueue(e)) +out["eligible"] = [r["event_id"] for r in c.delivery.eligible(now=c.clock.now())] diff --git a/internal/relay/delivery/testdata/scenarios/del02.py b/internal/relay/delivery/testdata/scenarios/del02.py new file mode 100644 index 00000000..35be7c66 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del02.py @@ -0,0 +1,4 @@ +rel = c.register() +p = c.ready_payload(rel, [c.artifact("out.txt", "still working")], turn=c.assigned_turn("inProgress")) +c.accept(p) +out["refused"] = refusal(c.delivery.enqueue, p["eventId"]) diff --git a/internal/relay/delivery/testdata/scenarios/del03.py b/internal/relay/delivery/testdata/scenarios/del03.py new file mode 100644 index 00000000..7c222fc0 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del03.py @@ -0,0 +1,28 @@ +import os +from codex_session_relay.models import Endpoint +from tests.support import HOST +def other(): + root = os.path.join(c.tmp, "other-project"); os.makedirs(root, exist_ok=True) + return c.registry.register(parent=Endpoint("01other-parent", HOST, cwd="/other", cxc_session="cxc-other"), + child=Endpoint("01other-child", HOST, cwd=root, cxc_session="cxc-other-c"), issue_key="REL-2", + artifact_roots=[root], allowed_recipients=["01other-parent"], dispatch_request_id="dispatch-2", dispatch_turn_id="turn-dispatch-2") +mode = sys.argv[3] if len(sys.argv) > 3 else "" +if mode == "scope": + _r, e = c.ready_event() + out["refused"] = refusal(c.delivery.enqueue, e, recipient_task_id="somebody-else") +elif mode == "other": + o = other() + _r, e = c.ready_event(recipients=[PARENT, o["parent"]["taskId"]]) + out["refused"] = refusal(c.delivery.enqueue, e, recipient_task_id=o["parent"]["taskId"]) + out["row"] = c.delivery.find(e) +elif mode == "tampered": + o = other() + _r, e = c.queued_event(recipients=[PARENT, o["parent"]["taskId"]]) + c.adapter.add_thread(o["parent"]["taskId"]) + with c.store.transaction() as db: + db.execute("UPDATE deliveries SET recipient_task_id = ?, recipient_thread_id = ? WHERE event_id = ?", (o["parent"]["taskId"], o["parent"]["taskId"], e)) + out["refused"] = refusal(c.attempt, e) +else: + other() + _r, e = c.queued_event() + out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del04.py b/internal/relay/delivery/testdata/scenarios/del04.py new file mode 100644 index 00000000..a046eeaf --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del04.py @@ -0,0 +1,3 @@ +_r, e = c.queued_event() +out["preview"] = c.delivery.render_message(e) +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del05.py b/internal/relay/delivery/testdata/scenarios/del05.py new file mode 100644 index 00000000..e6934b42 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del05.py @@ -0,0 +1,5 @@ +_r, e = c.queued_event() +t = c.adapter.start_turn(PARENT, status="inProgress") +c.adapter.set_status(PARENT, "active") +out["record"] = c.attempt(e) +out["turn"] = c.adapter.read_turn(PARENT, t.turn_id).status diff --git a/internal/relay/delivery/testdata/scenarios/del06.py b/internal/relay/delivery/testdata/scenarios/del06.py new file mode 100644 index 00000000..fcdaef1b --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del06.py @@ -0,0 +1,3 @@ +_r, e = c.queued_event() +c.adapter.script("busy") +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del07.py b/internal/relay/delivery/testdata/scenarios/del07.py new file mode 100644 index 00000000..4ae77d48 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del07.py @@ -0,0 +1,5 @@ +_r, e = c.queued_event() +if len(sys.argv) > 3: + c.adapter.start_turn(PARENT, turn_id="already-running", status="inProgress") + c.adapter.script("steer_existing") +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del08.py b/internal/relay/delivery/testdata/scenarios/del08.py new file mode 100644 index 00000000..8875015c --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del08.py @@ -0,0 +1,4 @@ +_r, e = c.queued_event() +c.attempt(e) +s = c.delivery.snapshot()["deliveries"][0] +out["snapshot"] = {k: s[k] for k in ("state", "reported", "acknowledged", "phase")} diff --git a/internal/relay/delivery/testdata/scenarios/del10.py b/internal/relay/delivery/testdata/scenarios/del10.py new file mode 100644 index 00000000..012ec06f --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del10.py @@ -0,0 +1,8 @@ +_r, e = c.queued_event() +c.adapter.threads[PARENT].approval_policy = sys.argv[3] +c.adapter.script("approval_policy") +out["record"] = c.attempt(e) +s = c.delivery.snapshot()["deliveries"][0] +out["snapshot"] = {k: s[k] for k in ("state", "reported", "holdReason", "phase")} +c.clock.advance(100000) +out["eligible"] = c.delivery.eligible(now=c.clock.now()) diff --git a/internal/relay/delivery/testdata/scenarios/del11.py b/internal/relay/delivery/testdata/scenarios/del11.py new file mode 100644 index 00000000..68efec3b --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del11.py @@ -0,0 +1,6 @@ +_r, e = c.queued_event() +c.adapter.script("busy") +first = c.attempt(e) +c.clock.advance(3600) +second = c.attempt(e, now=c.clock.now()) +out["first"], out["second"] = first, second diff --git a/internal/relay/delivery/testdata/scenarios/del12.py b/internal/relay/delivery/testdata/scenarios/del12.py new file mode 100644 index 00000000..64498d2b --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del12.py @@ -0,0 +1,12 @@ +_r, e = c.queued_event() +if sys.argv[3] == "dispatched": + c.attempt(e) + c.clock.advance(100000) + out["again"] = c.attempt(e, now=c.clock.now()) +else: + c.adapter.script("turn_start_fail") + out["first"] = c.attempt(e) + out["rounds"] = [] + for _ in range(5): + c.clock.advance(86400) + out["rounds"].append([c.delivery.eligible(now=c.clock.now()), c.attempt(e, now=c.clock.now())]) diff --git a/internal/relay/delivery/testdata/scenarios/del13.py b/internal/relay/delivery/testdata/scenarios/del13.py new file mode 100644 index 00000000..3ed6d54c --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del13.py @@ -0,0 +1,28 @@ +_r, e = c.queued_event() +mode = sys.argv[3] +if mode == "direct": + out["records"] = [] + for _ in range(c.delivery.policy.max_attempts): + c.adapter.script("read_fail"); c.clock.advance(100000) + out["records"].append(c.attempt(e, now=c.clock.now())) + c.clock.advance(100000) + out["eligible"] = c.delivery.eligible(now=c.clock.now()) +elif mode == "reconciled": + out["outcomes"] = [] + for _ in range(c.delivery.policy.max_attempts): + c.adapter.script("in_progress"); c.clock.advance(100000) + record = c.attempt(e, now=c.clock.now()) + if record is None: + break + c.adapter.ledger[record["requestId"]] = {"requestId": record["requestId"], "status": "failed", "error": "thread/read: refused", "rpcError": {"code": "internal", "message": "refused"}} + out["outcomes"].append(c.reconciler.reconcile_attempt(record["requestId"], c.adapter, now=c.clock.now())) +elif mode == "interval": + c.adapter.script("read_fail") + c.attempt(e) + c.delivery._reschedule(e, "withheld_pre_send", c.clock.now(), attempts=c.delivery_row(e)["attempt_count"]) + out["again"] = c.attempt(e, now=c.clock.now() + 1) +else: + now = c.clock.now() + with c.store.transaction() as db: + db.execute("INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,?,?)", (PARENT, int(now // 3600) * 3600, c.delivery.policy.max_sends_per_recipient_per_hour, 0)) + out["again"] = c.attempt(e, now=now) diff --git a/internal/relay/delivery/testdata/scenarios/del14.py b/internal/relay/delivery/testdata/scenarios/del14.py new file mode 100644 index 00000000..e3c6b469 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del14.py @@ -0,0 +1,11 @@ +_r, e = c.queued_event() +t = c.adapter.threads[PARENT] +mode = sys.argv[3] +if mode == "archived": t.archived = True +elif mode == "paused": t.goal_status = "paused" +elif mode == "budget": t.goal_status = "budgetLimited" +elif mode == "noinput": t.can_accept_input = False +elif mode == "idle": t.archived = True; t.status = "idle" +elif mode == "unreadable": c.adapter.fail_reads("read_goal_status") +out["status"] = c.registry.get(c._rid)["status"] +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del16.py b/internal/relay/delivery/testdata/scenarios/del16.py new file mode 100644 index 00000000..feef8ef8 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del16.py @@ -0,0 +1,9 @@ +_r, e = c.queued_event() +t = c.adapter.threads[PARENT] +if sys.argv[3] == "paused": + t.goal_status = "paused"; c.attempt(e); t.goal_status = "active" +else: + t.archived = True; c.attempt(e); t.archived = False +c.clock.advance(c.delivery.policy.lifecycle_recheck_seconds + 1) +out["eligible"] = [r["event_id"] for r in c.delivery.eligible(now=c.clock.now())] +out["record"] = c.attempt(e, now=c.clock.now()) diff --git a/internal/relay/delivery/testdata/scenarios/del17.py b/internal/relay/delivery/testdata/scenarios/del17.py new file mode 100644 index 00000000..c9eb658b --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del17.py @@ -0,0 +1,15 @@ +_r, e = c.queued_event() +mode = sys.argv[3] +if mode == "before": + c.registry.set_status(c._rid, "paused", actor="user") + out["eligible"] = c.delivery.eligible(now=c.clock.now()) +else: + original = c.adapter.read_goal_status + def hook(thread_id): + if mode == "supersede": + c.registry.supersede(c._rid, new_relationship_id="rel-bbbbbbbbbbbbbbbb") + else: + c.registry.set_status(c._rid, mode, actor="user") + return original(thread_id) + c.adapter.read_goal_status = hook + out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del18.py b/internal/relay/delivery/testdata/scenarios/del18.py new file mode 100644 index 00000000..9bb927e8 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del18.py @@ -0,0 +1,3 @@ +_r, e = c.queued_event() +c.registry.set_status(c._rid, sys.argv[3], actor="user") +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del20.py b/internal/relay/delivery/testdata/scenarios/del20.py new file mode 100644 index 00000000..c8441104 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del20.py @@ -0,0 +1,10 @@ +_r, e = c.queued_event() +if sys.argv[3] == "resume": + c.registry.set_status(c._rid, "paused", actor="user") + c.attempt(e) + rec = c.registry.get(c._rid) + c.registry.resume(c._rid, expect_generation=rec["executionGeneration"], expect_artifact_roots=rec["authorizedScope"]["artifactRoots"], expect_allowed_recipients=rec["authorizedScope"]["allowedRecipients"], actor="user") + c.clock.advance(c.delivery.policy.lifecycle_recheck_seconds + 1) + out["record"] = c.attempt(e, now=c.clock.now()) +else: + out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/del21.py b/internal/relay/delivery/testdata/scenarios/del21.py new file mode 100644 index 00000000..819c247f --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del21.py @@ -0,0 +1,17 @@ +_r, e = c.queued_event() +mode = sys.argv[3] +if mode in ("running", "shorten"): + c.adapter.set_status(PARENT, "active") + out["first"] = c.attempt(e) + out["deferred_until"] = c.delivery_row(e)["next_eligible_at"] + c.registry.set_status(c._rid, "cancelled", actor="user") + out["record"] = c.attempt(e) + out["after"] = c.delivery_row(e)["next_eligible_at"] +else: + c.registry.set_status(c._rid, "cancelled", actor="user") + stale = c.registry.get(c._rid) + far = c.clock.now() + 100000 + with c.store.transaction() as db: + db.execute("UPDATE deliveries SET next_eligible_at = ? WHERE event_id = ?", (far, e)) + out["record"] = c.delivery._withhold_inactive(e, stale, c.clock.now(), attempts=0) + out["after"] = c.delivery_row(e)["next_eligible_at"] diff --git a/internal/relay/delivery/testdata/scenarios/del22.py b/internal/relay/delivery/testdata/scenarios/del22.py new file mode 100644 index 00000000..d2d10f99 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del22.py @@ -0,0 +1,7 @@ +_r, e = c.queued_event() +if sys.argv[3] == "superseded": + c.registry.supersede(c._rid, new_relationship_id="rel-bbbbbbbbbbbbbbbb") + out["record"] = c.attempt(e) +else: + stale = dict(c.registry.get(c._rid)); stale["status"] = "paused" + out["record"] = c.delivery._withhold_inactive(e, stale, c.clock.now(), attempts=0) diff --git a/internal/relay/delivery/testdata/scenarios/del24.py b/internal/relay/delivery/testdata/scenarios/del24.py new file mode 100644 index 00000000..39e032a8 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del24.py @@ -0,0 +1,15 @@ +_r, e = c.queued_event() +if sys.argv[3] == "busy": + out["record"] = c.attempt(e) + row = c.delivery_row(e) + c.delivery._defer_busy(e, dict(row, state="queued", attempt_count=0), c.clock.now()) + c.clock.advance(100000) + out["again"] = c.attempt(e, now=c.clock.now()) +else: + c.adapter.script("busy") + first = c.attempt(e) + c.clock.advance(3600) + out["second"] = c.attempt(e, now=c.clock.now()) + out["reconciled"] = c.reconciler.reconcile_attempt(first["requestId"], c.adapter) + c.clock.advance(100000) + out["again"] = c.attempt(e, now=c.clock.now()) diff --git a/internal/relay/delivery/testdata/scenarios/del25.py b/internal/relay/delivery/testdata/scenarios/del25.py new file mode 100644 index 00000000..eb4a8101 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del25.py @@ -0,0 +1,7 @@ +_r, e = c.queued_event() +c.adapter.script("in_progress") +record = c.attempt(e) +turn = c.adapter.start_turn(PARENT, status="inProgress") +c.adapter.ledger[record["requestId"]] = {"requestId": record["requestId"], "status": "accepted", "resumed": {"approvalPolicy": "never"}, "turnId": turn.turn_id} +out["record"] = record +out["reconciled"] = c.reconciler.reconcile_attempt(record["requestId"], c.adapter) diff --git a/internal/relay/delivery/testdata/scenarios/del26.py b/internal/relay/delivery/testdata/scenarios/del26.py new file mode 100644 index 00000000..afc05ccc --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del26.py @@ -0,0 +1,18 @@ +rel = c.register() +p = c.ready_payload(rel, [c.artifact("out.txt", "finished after several turns")], turn=c.assigned_turn("completed", turn="turn-loop-3")) +obs = c.assigned_turn("completed", turn="turn-loop-3") +mode = sys.argv[3] +if mode == "none": + out["result"] = refusal(c.accept, p) +elif mode == "valid": + out["result"] = refusal(c.intake.accept_child_receipt, p, observation=obs, continuation={"anchorTurnId": "turn-dispatch-1", "actor": "child-loop", "reason": "PABCD cycle 3 completed this generation"}) +elif mode == "anchor": + out["result"] = refusal(c.intake.accept_child_receipt, p, observation=obs, continuation={"anchorTurnId": "some-other-execution", "actor": "a", "reason": "b"}) +elif mode == "thread": + p = c.ready_payload(rel, [c.artifact("out.txt", "payload")], turn=c.assigned_turn("completed", thread="someone-else", turn="turn-loop-3")) + out["result"] = refusal(c.intake.accept_child_receipt, p, observation=c.assigned_turn("completed", thread="someone-else", turn="turn-loop-3"), continuation={"anchorTurnId": "turn-dispatch-1", "actor": "a", "reason": "b"}) +elif mode == "malformed": + out["result"] = refusal(c.intake.accept_child_receipt, p, observation=obs, continuation={"anchorTurnId": "turn-dispatch-1"}) +else: + c.intake.accept_child_receipt(p, observation=obs, continuation={"anchorTurnId": "turn-dispatch-1", "actor": "a", "reason": "b"}) + out["result"] = refusal(c.intake.accept_child_receipt, p, observation=obs) diff --git a/internal/relay/delivery/testdata/scenarios/del27.py b/internal/relay/delivery/testdata/scenarios/del27.py new file mode 100644 index 00000000..2a8f4358 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del27.py @@ -0,0 +1,17 @@ +from codex_session_relay import identity +mode = sys.argv[3] +if mode == "completion": + _r, e = c.queued_event() + out["message"] = c.delivery.render_message(e) +else: + _r, e = c.queued_event(recipients=[PARENT, CHILD]) + c.attempt(e) + c.clock.advance(5) + turn = c.adapter.start_turn(PARENT, turn_id="ack-turn", status="inProgress") + out["ack"] = c.ack.acknowledge(e, ack_turn_id=turn.turn_id, ack_proof=identity.ack_proof(e, turn.turn_id), accepted=True, adapter=c.adapter) + out["verdict"] = c.ack.record_verdict(e, verdict="needs_changes", verdict_turn_id="verdict-1", criteria=[{"id": "c-1", "verdict": "needs_changes", "note": "the manifest omits the migration script"}]) + rev = c.store.one("SELECT * FROM deliveries WHERE kind = 'revision_request'")["event_id"] + out["revision"] = rev + out["message"] = c.delivery.render_message(rev) + out["childAck"] = refusal(c.ack.acknowledge, rev, ack_turn_id="child-turn", ack_proof=identity.ack_proof(rev, "child-turn"), accepted=True, adapter=c.adapter) + out["generation"] = c.registry.get(c._rid)["executionGeneration"] diff --git a/internal/relay/delivery/testdata/scenarios/del30.py b/internal/relay/delivery/testdata/scenarios/del30.py new file mode 100644 index 00000000..4a52c351 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del30.py @@ -0,0 +1,8 @@ +import os +from codex_session_relay.models import Endpoint +from codex_session_relay.registry import project_key +from tests.support import HOST +root = os.path.join(c.tmp, "other-project"); os.makedirs(root, exist_ok=True) +other = c.registry.register(parent=Endpoint("01other-parent", HOST, cwd="/other", cxc_session="cxc-other"), child=Endpoint("01other-child", HOST, cwd=root, cxc_session="cxc-other-c"), issue_key="REL-2", artifact_roots=[root], allowed_recipients=["01other-parent"], dispatch_request_id="dispatch-2", dispatch_turn_id="turn-dispatch-2") +mine = c.register(issue_key="REL-3", dispatch_request_id="dispatch-3") +out["mine"], out["other"] = project_key(mine), project_key(other) diff --git a/internal/relay/delivery/testdata/scenarios/del31.py b/internal/relay/delivery/testdata/scenarios/del31.py new file mode 100644 index 00000000..1c2f15ae --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del31.py @@ -0,0 +1,25 @@ +from codex_session_relay.sync import SyncOutbox +from codex_session_relay.store import Store +from codex_session_relay.registry import Registry +from codex_session_relay.receipts import ReceiptIntake +from codex_session_relay.delivery import DeliveryService +from codex_session_relay.reconcile import Reconciler +rel, e1 = c.queued_event() +out["first"] = c.attempt(e1) +p = c.ready_payload(rel, [c.artifact("second.txt", "still in flight")], attempt=2) +c.accept(p) +e2 = p["eventId"] +c.delivery.enqueue(e2) +c.adapter.script("transport_unknown") +later = c.clock.now() + 3600 +out["second"] = c.attempt(e2, now=later) +SyncOutbox(c.store, c.clock).set_target(rel["relationshipId"], "coordination_document", "DOC-1") +path = c.store.path +c.store.close() +c.store = Store(path) +c.registry = Registry(c.store, c.clock) +c.intake = ReceiptIntake(c.store, c.registry, c.clock) +c.delivery = DeliveryService(c.store, c.registry, c.intake, c.clock) +c.reconciler = Reconciler(c.store, c.registry, c.delivery, c.clock) +out["recovered"] = c.reconciler.recover_on_start(c.adapter) +out["again"] = c.attempt(e1, now=later + 3600) diff --git a/internal/relay/delivery/testdata/scenarios/del32.py b/internal/relay/delivery/testdata/scenarios/del32.py new file mode 100644 index 00000000..80f237b0 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del32.py @@ -0,0 +1,7 @@ +import json +_r, e = c.queued_event() +row = c.store.one("SELECT receipt FROM events WHERE event_id = ?", (e,)) +receipt = json.loads(row["receipt"]) +receipt["criteria"] = [{"id": "c1", "verdict": "verified", "restoration": "false"}, {"id": "c2", "verdict": "verified", "restoration": True}] +c.store.db.execute("UPDATE events SET receipt = ? WHERE event_id = ?", (json.dumps(receipt), e)) +out["message"] = c.delivery.preview_message(e) diff --git a/internal/relay/delivery/testdata/scenarios/del33.py b/internal/relay/delivery/testdata/scenarios/del33.py new file mode 100644 index 00000000..a89e6731 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del33.py @@ -0,0 +1,9 @@ +_r, e = c.queued_event() +now = c.clock.now() +c.store.db.execute("INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,1,?)", (PARENT, int(now // 3600) * 3600, now)) +c.store.db.commit() +c.delivery._rate_limited = lambda recipient, at: False +out["first"] = c.attempt(e, now=now + 1) +out["row"] = dict(c.delivery_row(e)) +del c.delivery._rate_limited +out["record"] = c.attempt(e, now=out["row"]["next_eligible_at"]) diff --git a/internal/relay/delivery/testdata/scenarios/del34.py b/internal/relay/delivery/testdata/scenarios/del34.py new file mode 100644 index 00000000..815d8b1c --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del34.py @@ -0,0 +1,17 @@ +from codex_session_relay.bridge_adapter import BridgeHostAdapter +from tests.test_bridge_adapter import FakeRpc, emulated_thread_list +from tests.test_delivery import ExecAwareHost +def host(archived=False): + threads = [{"id": PARENT, "source": "vscode"}, {"id": CHILD, "source": "exec", "archived": archived}] + return ExecAwareHost(c.adapter, BridgeHostAdapter(call=FakeRpc({"thread/list": emulated_thread_list(threads)}), store=c.store, clock=c.clock)) +_completion, correction = c.correction_after_needs_changes() +out["correction"] = correction +mode = sys.argv[3] +if mode == "live": + out["record"] = c.delivery.attempt(correction, host()) +else: + if mode == "legacy": + c.store.db.execute("INSERT INTO failed_operations (scope_key, operation, detail, error_code, occurred_at) VALUES (?, 'lifecycle_read', 'lifecycle_unknown', 'lifecycle_unknown', ?)", (correction, c.clock.iso())) + c.store.db.commit() + c.clock.advance(1) + out["record"] = c.delivery.attempt(correction, host(True)) diff --git a/internal/relay/delivery/testdata/scenarios/del35.py b/internal/relay/delivery/testdata/scenarios/del35.py new file mode 100644 index 00000000..f0d5f9ff --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/del35.py @@ -0,0 +1,21 @@ +mode = sys.argv[3] +_r, e = c.queued_event(settings=None) if mode == "settings" else c.queued_event() +if mode == "stamp": + c.adapter.threads[PARENT].archived = True + original = c.clock.iso + def ticking(): + c.clock.advance(0.000001) + return original() + c.clock.iso = ticking +else: + if mode == "lifecycle": + c.adapter.fail_reads("is_archived") + elif mode == "busy": + c.adapter.threads[PARENT].status = "active" + original = c.adapter.read_goal_status + def claimed(thread_id): + c.store.db.execute("UPDATE deliveries SET state = 'sending', attempt_count = attempt_count + 1 WHERE event_id = ?", (e,)) + c.store.db.commit() + return original(thread_id) + c.adapter.read_goal_status = claimed +out["record"] = c.attempt(e) diff --git a/internal/relay/delivery/testdata/scenarios/drl.py b/internal/relay/delivery/testdata/scenarios/drl.py new file mode 100644 index 00000000..ab2bfb22 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/drl.py @@ -0,0 +1,14 @@ +from codex_session_relay.delivery import DeliveryService +cases = json.load(open(os.path.join(os.path.dirname(__file__), "drl_answers.json"))) +rel = {"relationshipId": "rel-1", "parent": {"taskId": "parent-task"}, "child": {"taskId": "child-task"}} +class Reader: + def __init__(self, answer): self.answer, self.asked = answer, [] + def up(self, **kw): self.asked.append(kw); return self.answer +for name, case in cases.items(): + reader = Reader(case["answer"]) + service = DeliveryService(c.store, c.registry, c.intake, c.clock, linkage=reader) + out[name] = refusal(lambda: list(service.resolve_recipient(rel, case["kind"]))) + out[name]["asked"] = reader.asked +bare = DeliveryService(c.store, c.registry, c.intake, c.clock) +out["unwired_completion"] = list(bare.resolve_recipient(rel, "completion_event")) +out["unwired_revision"] = list(bare.resolve_recipient(rel, "revision_request")) diff --git a/internal/relay/delivery/testdata/scenarios/mpi.py b/internal/relay/delivery/testdata/scenarios/mpi.py new file mode 100644 index 00000000..8f0622c7 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/mpi.py @@ -0,0 +1,84 @@ +import threading +from codex_session_relay import identity +from codex_session_relay.ack import AckService +from codex_session_relay.delivery import DeliveryService, REVISION +from codex_session_relay.models import Endpoint +from codex_session_relay.receipts import ReceiptIntake +from codex_session_relay.registry import Registry, project_key, record_settings +from codex_session_relay.store import Store +from codex_session_relay.sync import SyncOutbox +from codex_session_relay.errors import RelayError +from tests.support import HOST, task_settings +PROJECTS = {"a": ("/repo-a", "AAA-1", "linear://project-alpha"), "b": ("/repo-b", "BBB-1", "linear://project-beta")} +ALPHA_DOC = "https://linear.app/example/document/project-alpha-0000" +BETA_DOC = "https://linear.app/example/document/project-beta-00000" +sync = SyncOutbox(c.store, c.clock); c.ack.sync = sync +def assignment(name, events=1): + cwd, iss, ref = PROJECTS[name] + par, chi = f"01parent-{name}", f"01child-{name}" + root = os.path.join(c.root, name); os.makedirs(root, exist_ok=True) + rel = c.registry.register(parent=Endpoint(par, HOST, cwd=cwd), child=Endpoint(chi, HOST, cwd=root), issue_key=iss, artifact_roots=[root], allowed_recipients=[par, chi], dispatch_request_id=f"dispatch-{name}", dispatch_turn_id=f"turn-{name}", scope_ref=ref) + c.adapter.add_thread(par); c.adapter.add_thread(chi) + record_settings(c.store, c.clock, par, task_settings(cwd), source="creation_result") + ids = [] + for i in range(events): + path = os.path.join(root, f"out-{i}.txt") + open(path, "w").write(f"{name}-{i}") + p = c.ready_payload(rel, [path], attempt=i + 1, turn=c.assigned_turn(thread=chi, turn=f"turn-{name}")) + c.accept(p); c.delivery.enqueue(p["eventId"]); ids.append(p["eventId"]); c.clock.advance(1) + return rel, ids +def deliver(e): + return c.delivery.attempt(e, c.adapter, now=c.clock.now()) +def acked(name, e): + deliver(e); c.clock.advance(5) + t = c.adapter.start_turn(f"01parent-{name}", turn_id=f"ack-{name}", status="inProgress") + return t +def parallel(work): + barrier = threading.Barrier(len(work)); results, errors = {}, {} + def wrap(name, call): + def run(): + store = Store(c.store.path) + try: + reg = Registry(store, c.clock); intake = ReceiptIntake(store, reg, c.clock) + d = DeliveryService(store, reg, intake, c.clock); a = AckService(store, reg, intake, d, c.clock); a.sync = SyncOutbox(store, c.clock) + barrier.wait(timeout=20); results[name] = call(a) + except Exception as error: + errors[name] = repr(error) + finally: + store.close() + return threading.Thread(target=run) + ts = [wrap(n, f) for n, f in work.items()] + [t.start() for t in ts]; [t.join(timeout=60) for t in ts] + return results, errors +m = sys.argv[3] +if m == "scope": + a, _ = assignment("a"); b, _ = assignment("b") + out["keys"] = [project_key(a), project_key(b)] +elif m == "acks": + a, ai = assignment("a"); b, bi = assignment("b") + turns = {"a": acked("a", ai[0]), "b": acked("b", bi[0])} + res, err = parallel({n: (lambda n, e: lambda s: s.acknowledge(e, ack_turn_id=turns[n].turn_id, ack_proof=identity.ack_proof(e, turns[n].turn_id), accepted=True, adapter=c.adapter))(n, e) for n, e in (("a", ai[0]), ("b", bi[0]))}) + out["errors"] = err +elif m == "verdicts": + from codex_session_relay import identity + a, ai = assignment("a"); b, bi = assignment("b") + for n, ids in (("a", ai), ("b", bi)): + t = acked(n, ids[0]) + c.ack.acknowledge(ids[0], ack_turn_id=t.turn_id, ack_proof=identity.ack_proof(ids[0], t.turn_id), accepted=True, adapter=c.adapter) + res, err = parallel({n: (lambda n, e: lambda s: s.record_verdict(e, verdict="needs_changes", verdict_turn_id=f"verdict-{n}", findings=[{"id": "c1", "verdict": "needs_changes", "note": "fix it"}]))(n, e) for n, e in (("a", ai[0]), ("b", bi[0]))}) + out["errors"] = err + out["next"] = {n: r["nextExecutionGeneration"] for n, r in res.items()} +elif m == "outbox": + from codex_session_relay import identity + jobs = {} + for n, target in (("a", ALPHA_DOC), ("b", BETA_DOC)): + rel, ids = assignment(n) + t = acked(n, ids[0]) + c.ack.acknowledge(ids[0], ack_turn_id=t.turn_id, ack_proof=identity.ack_proof(ids[0], t.turn_id), accepted=True, adapter=c.adapter) + sync.set_target(rel["relationshipId"], "coordination_document", target) + c.ack.record_verdict(ids[0], verdict="verified", verdict_turn_id=f"verdict-{n}") + jobs[n] = sync.snapshot(relationship_id=rel["relationshipId"])["jobs"][0]["syncId"] + out["jobs"] = jobs + alpha = sync.claim(jobs["a"], owner="worker-1", now=c.clock.now()) + sync.claim(jobs["b"], owner="worker-1", now=c.clock.now()) + out["complete"] = refusal(sync.complete, jobs["b"], claim_token=alpha["claimToken"], target_ref=BETA_DOC, readback="", now=c.clock.now()) diff --git a/internal/relay/delivery/testdata/scenarios/ogi.py b/internal/relay/delivery/testdata/scenarios/ogi.py new file mode 100644 index 00000000..1a6dd5c6 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/ogi.py @@ -0,0 +1,8 @@ +_r, e = c.queued_event() +rid = c._rid +with c.store.transaction() as db: + out["first"] = c.registry.open_generation_in(db, rid, dispatch_request_id="revision-x", reason="needs_changes_revision") +with c.store.transaction() as db: + out["replay"] = c.registry.open_generation_in(db, rid, dispatch_request_id="revision-x", reason="needs_changes_revision") +with c.store.transaction() as db: + out["next"] = c.registry.open_generation_in(db, rid, dispatch_request_id="revision-y", reason="needs_changes_revision") diff --git a/internal/relay/delivery/testdata/scenarios/ord.py b/internal/relay/delivery/testdata/scenarios/ord.py new file mode 100644 index 00000000..9a26a265 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/ord.py @@ -0,0 +1,41 @@ +import dataclasses +from codex_session_relay import identity +from codex_session_relay.settings import TaskSettings +from codex_session_relay.delivery import DeliveryService +from tests.support import task_settings +def on_request(path="/parent"): + return task_settings(path, approvalPolicy="on-request") +m = sys.argv[3] +if m == "record": + out["onRequest"] = refusal(TaskSettings(on_request()).require_usable) + out["refused"] = [refusal(TaskSettings(task_settings("/parent", approvalPolicy=r)).require_usable) for r in ("untrusted", {"granular": {}}, None)] + out["resume"] = [TaskSettings(r).resume_params("t-1") for r in (task_settings("/parent"), on_request())] +elif m == "woken": + _r, e = c.queued_event(settings=on_request()) + c.adapter.threads[PARENT].approval_policy = "on-request" + out["record"] = c.attempt(e) + c.clock.advance(100000) + out["eligible"] = c.delivery.eligible(now=c.clock.now()) +elif m == "busy": + _r, e = c.queued_event(settings=on_request()) + c.adapter.threads[PARENT].approval_policy = "on-request" + c.adapter.script("busy") + out["busy"] = c.attempt(e) + row = c.delivery_row(e) + out["record"] = c.attempt(e, now=row["next_eligible_at"]) +elif m == "folded": + _r, e = c.queued_event(settings=on_request()) + c.adapter.threads[PARENT].approval_policy = "on-request" + existing = c.adapter.start_turn(PARENT, status="inProgress") + c.adapter.script("steer_existing") + out["record"] = c.attempt(e) + proof = identity.ack_proof(e, existing.turn_id) + out["acks"] = [c.ack.acknowledge(e, ack_turn_id=existing.turn_id, ack_proof=proof, accepted=True, adapter=c.adapter) for _ in range(2)] + turns = c.adapter.threads[PARENT].turns + turns[turns.index(existing)] = dataclasses.replace(existing, status="completed") + c.adapter.restart() + for thread in c.adapter.threads.values(): + thread.status = "notLoaded" + restarted = DeliveryService(c.store, c.registry, c.intake, c.clock) + c.clock.advance(100000) + out["after"] = restarted.attempt(e, c.adapter, now=c.clock.now()) diff --git a/internal/relay/delivery/testdata/scenarios/qa.py b/internal/relay/delivery/testdata/scenarios/qa.py new file mode 100644 index 00000000..e23361ee --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/qa.py @@ -0,0 +1,9 @@ +from codex_session_relay import identity +rel, e = c.queued_event() +out["delivered"] = c.attempt(e) +c.clock.advance(5) +t = c.adapter.start_turn(PARENT, turn_id="ack-turn", status="inProgress") +out["claim"] = c.ack.claim_verification(e, turn_id="ack-turn") +out["ack"] = c.ack.acknowledge(e, ack_turn_id=t.turn_id, ack_proof=identity.ack_proof(e, t.turn_id), accepted=True, adapter=c.adapter) +out["verdict"] = c.ack.record_verdict(e, verdict="verified", verdict_turn_id="verdict-1") +out["duplicate"] = refusal(c.accept, c.ready_payload(rel, [c.artifact("out.txt", "the deliverable")])) diff --git a/internal/relay/delivery/testdata/scenarios/rvr.py b/internal/relay/delivery/testdata/scenarios/rvr.py new file mode 100644 index 00000000..2e1630b7 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/rvr.py @@ -0,0 +1,75 @@ +import json +from codex_session_relay.identity import ack_proof +from codex_session_relay.currency import head_revision +exec(open(os.path.join(os.path.dirname(__file__), "scenarios", "_vcu.py")).read()) +def request_correction(e): + result = c.ack.record_verdict(e, verdict="needs_changes", verdict_turn_id=f"review-{e}", findings=[{"id": "c1", "verdict": "needs_changes", "note": "fix the output"}]) + g = result["nextExecutionGeneration"] + req = c.store.one("SELECT * FROM events WHERE relationship_id = ? AND execution_generation = ? AND outcome = 'revision_request'", (c._rid, g)) + c.attempt(req["event_id"]) + bound = c.ack.bind_dispatched_revision(req["event_id"]) + c.adapter.finish_turn(CHILD, bound["dispatchTurnId"]) + return json.loads(req["receipt"]) +def emit(pred, text="corrected output"): + rel = c.registry.get(c._rid) + g = c.registry.generation(c._rid, rel["executionGeneration"]) + turn = c.assigned_turn(turn=g["dispatchTurnId"]) + p = c.ready_payload(rel, [c.artifact("out.txt", text)], turn=turn) + c.intake.accept_child_receipt(p, observation=turn, supersedes_revision=pred) + return p +def ack_correction(p): + e = p["eventId"]; c.delivery.enqueue(e); c.attempt(e); c.clock.advance(5) + t = c.adapter.start_turn(PARENT, status="inProgress") + c.ack.acknowledge(e, ack_turn_id=t.turn_id, ack_proof=ack_proof(e, t.turn_id), accepted=True, adapter=c.adapter) +def headof(g=2): + return head_revision(c.store.db, c._rid, g) +m = sys.argv[3] +if m == "roundtrip": + first = acknowledged(); req = request_correction(first); corr = emit(req["supersedesRevisionHash"]); ack_correction(corr) + out["final"] = verdict(corr["eventId"], "verified", turn="review-corrected", findings=[{"id": "c1", "verdict": "verified"}]) + out["head"] = headof() + out["replay"] = verdict(first, "needs_changes", turn="replayed-review") + out["generation"] = c.registry.get(c._rid)["executionGeneration"] +elif m == "repeated": + first = acknowledged(); req = request_correction(first) + s = emit(req["supersedesRevisionHash"], "second output"); ack_correction(s) + req2 = request_correction(s["eventId"]); th = emit(req2["supersedesRevisionHash"], "third output"); ack_correction(th) + out["final"] = verdict(th["eventId"], "verified", turn="review-third", findings=[{"id": "c1", "verdict": "verified"}]) +elif m == "extend": + req = request_correction(acknowledged()); corr = emit(req["supersedesRevisionHash"], "first fix") + latest = emit(corr["revisionHash"], "refined fix"); ack_correction(latest) + out["final"] = verdict(latest["eventId"], "verified", turn="final-review", findings=[{"id": "c1", "verdict": "verified"}]) +elif m == "historical": + first = acknowledged(); advance(); emit(c.intake.row(first)["revision_hash"]); out["head"] = headof() +elif m == "unknown": + request_correction(acknowledged()); emit("f" * 64); out["head"] = headof() +elif m == "earlier": + first = acknowledged(); fh = c.intake.row(first)["revision_hash"] + s = emit(fh, "successor"); ack_correction(s); request_correction(s["eventId"]); emit(fh); out["head"] = headof() +elif m == "other_rel": + first = acknowledged(text="first relationship output"); fh = c.intake.row(first)["revision_hash"] + request_correction(first) + other = c.register(issue_key="REL-OTHER", dispatch_request_id="other-assignment", recipients=[PARENT, CHILD]) + c._rid = other["relationshipId"] + init = emit(None, "other relationship output"); ack_correction(init); request_correction(init["eventId"]); emit(fh) + out["head"] = headof() +elif m == "suppressed": + req = request_correction(acknowledged()) + with c.store.transaction() as db: + db.execute("UPDATE events SET suppressed_reason = 'withdrawn' WHERE event_id = ?", (req["eventId"],)) + emit(req["supersedesRevisionHash"]); out["head"] = headof() +elif m == "fork": + req = request_correction(acknowledged()); emit(req["supersedesRevisionHash"], "one"); emit(req["supersedesRevisionHash"], "two") + out["head"] = headof() +elif m == "old_unruled": + first = acknowledged(); s = emit(c.intake.row(first)["revision_hash"], "successor"); ack_correction(s) + req = request_correction(s["eventId"]); emit(req["supersedesRevisionHash"]) + out["r"] = verdict(first, "verified", turn="late-review") +elif m == "projection": + first = acknowledged(); req = request_correction(first) + out["request"] = req + if len(sys.argv) > 4: + out["corrected"] = emit(req["supersedesRevisionHash"]) + from codex_session_relay.assignment import AssignmentView + proj = AssignmentView(c.store, c.registry, c.clock).state(c._rid)["projection"] + out["projection"] = {k: proj[k] for k in ("completion", "correction")} diff --git a/internal/relay/delivery/testdata/scenarios/sup.py b/internal/relay/delivery/testdata/scenarios/sup.py new file mode 100644 index 00000000..8c4dfc47 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/sup.py @@ -0,0 +1,95 @@ +from codex_session_relay.models import TurnRef +def advance(number): + t = f"turn-dispatch-{number}" + c.adapter.start_turn(CHILD, turn_id=t, status="inProgress") + return c.registry.open_generation(c._rid, dispatch_request_id=f"dispatch-{number}", reason="needs_changes_revision", dispatch_turn_id=t) +def queued_outcome(outcome): + rel = c.register(); c._rid = rel["relationshipId"] + p = c.ready_payload(rel, [c.artifact("out.txt", "the deliverable")]) if outcome == "ready_for_review" else c.execution_payload(rel, outcome) + c.accept(p); c.delivery.enqueue(p["eventId"]) + return rel, p["eventId"] +def declare(successor, older): + with c.store.transaction() as db: + db.execute("UPDATE revision_lineage SET supersedes_hash = ? WHERE event_id = ?", (c.intake.get(older)["revisionHash"], successor)) +def item(e): + s = [d for d in c.delivery.snapshot()["deliveries"] if d["eventId"] == e][0] + return {k: s[k] for k in ("state", "reported", "phase", "supersededNote", "holdReason")} +m = sys.argv[3] +if m in ("queued", "outstanding", "capped", "withheld_cap", "dispatched", "current", "mark"): + rel, e = queued_outcome("ready_for_review") + if m in ("outstanding", "current", "mark"): + c.adapter.script("transport_unknown") + if m == "mark": c.clock.advance(3600) + c.attempt(e) + elif m == "capped": + with c.store.transaction() as db: db.execute("UPDATE deliveries SET state = ?, hold_reason = ? WHERE event_id = ?", ("deferred_busy", "busy_cap", e)) + elif m == "withheld_cap": + with c.store.transaction() as db: db.execute("UPDATE deliveries SET state = ?, hold_reason = ? WHERE event_id = ?", ("withheld_pre_send", "presend_cap", e)) + elif m == "dispatched": + c.attempt(e) + advance(2) + if m == "current": + c.store.db.execute("DELETE FROM delivery_supersession WHERE event_id = ?", (e,)) + advance(3) + if m == "mark": + c.delivery.mark_superseded(e, reason="stale_generation") + out["item"] = item(e) +elif m == "terminal" or m == "queued_pred": + rel, older = queued_outcome("ready_for_review") + if m == "terminal": + c.adapter.script("transport_unknown"); c.attempt(older) + s = c.ready_payload(rel, [c.artifact("newer.txt", "the corrected deliverable")], attempt=2) + c.accept(s); declare(s["eventId"], older) + c.delivery.enqueue(s["eventId"]) + out["item"] = item(older) +elif m == "outstanding_pred": + rel, older = queued_outcome("ready_for_review") + c.adapter.start_turn(CHILD, turn_id="turn-dispatch-1", status="inProgress") + c.adapter.script("transport_unknown"); c.attempt(older) + s = c.ready_payload(rel, [c.artifact("newer.txt", "the corrected deliverable")], attempt=2, turn=TurnRef(CHILD, "turn-dispatch-1", "inProgress")) + c.accept(s); declare(s["eventId"], older) + c.intake.resolve_staged(TurnRef(CHILD, "turn-dispatch-1", "completed")) + c.delivery.annotate_predecessors(s["eventId"]) + out["item"] = item(older) +elif m == "reemit": + rel = c.register() + p = c.ready_payload(rel, [c.artifact("out.txt", "the deliverable")]) + out["first"] = c.accept(p); out["again"] = c.accept(p) +elif m == "exec_only": + rel, reviewable = queued_outcome("ready_for_review") + c.attempt(reviewable) + later = c.execution_payload(rel, "failed") + c.accept(later); c.delivery.enqueue(later["eventId"]) + with c.store.transaction() as db: + out["reason"] = c.delivery._supersession_reason(db, later["eventId"]) + out["state"] = c.delivery.get(later["eventId"])["state"] +elif m in ("intent", "stranded"): + rel, e = queued_outcome("ready_for_review") + if m == "intent": + c.store.db.execute("DELETE FROM deliveries WHERE event_id = ?", (e,)) + with c.store.transaction() as db: + c.delivery.record_intent_in(db, e, relationship_id=rel["relationshipId"], kind="completion", recipient_task_id=PARENT, error=RuntimeError("the relationship was paused"), now=c.clock.now()) + out["row"] = dict(c.delivery.enqueue(e)) +elif m.startswith("presend_"): + outcome = m[len("presend_"):] + rel, e = queued_outcome(outcome) + advance(2); c.clock.advance(3600) + out["record"] = c.attempt(e) +elif m == "busy_release": + rel, e = queued_outcome("ready_for_review") + c.adapter.set_status(PARENT, "active") + out["first"] = c.attempt(e) + advance(2); c.adapter.set_status(PARENT, "idle"); c.clock.advance(3600) + out["record"] = c.attempt(e) +elif m == "newer": + rel, older = queued_outcome("ready_for_review") + n = c.ready_payload(rel, [c.artifact("newer.txt", "the corrected deliverable")], attempt=2) + c.accept(n); declare(n["eventId"], older) + c.delivery.enqueue(n["eventId"]); c.clock.advance(3600) + out["record"] = c.attempt(older); c.clock.advance(3600) + out["sent"] = c.attempt(n["eventId"]) +elif m == "staged_successor": + rel, older = queued_outcome("ready_for_review") + st = c.ready_payload(rel, [c.artifact("staged.txt", "still being written")], attempt=2, turn=TurnRef(CHILD, "turn-dispatch-1", "inProgress")) + c.accept(st); c.clock.advance(3600) + out["record"] = c.attempt(older) diff --git a/internal/relay/delivery/testdata/scenarios/vcu.py b/internal/relay/delivery/testdata/scenarios/vcu.py new file mode 100644 index 00000000..fac8de21 --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/vcu.py @@ -0,0 +1,63 @@ +exec(open(os.path.join(os.path.dirname(__file__), "scenarios", "_vcu.py")).read()) +m = sys.argv[3] +if m == "advanced": + e = acknowledged(); advance(); out["r"] = verdict(e, "verified") +elif m == "ambiguous": + e = acknowledged(); second(); out["r"] = verdict(e, "verified") +elif m == "superseded": + e = acknowledged(); supersede(e, "the corrected revision", c.intake.row(e)["revision_hash"]); out["r"] = verdict(e, "verified") +elif m == "paused": + e = acknowledged(); c.registry.set_status(c._rid, "paused", actor=PARENT) + out["r"] = [verdict(e, v, reason="stopping") for v in ("verified", "needs_changes", "unverified", "aborted")] +elif m == "nc_stale": + e = acknowledged(); advance(); out["r"] = verdict(e, "needs_changes", findings=[{"id": "c1", "verdict": "needs_changes", "note": "still wrong"}]) +elif m == "unv_stale": + e = acknowledged(); advance(); out["r"] = verdict(e, "unverified", reason="could not reach it") +elif m == "replay": + e = acknowledged() + out["first"] = verdict(e, "needs_changes", findings=[{"id": "c1", "verdict": "needs_changes", "note": "fix it"}]) + out["r"] = verdict(e, "verified", turn="v2") +elif m == "sole": + acknowledged(); out["r"] = head() +elif m == "chain": + e = acknowledged(); supersede(e, "second revision", c.intake.row(e)["revision_hash"]); out["r"] = head() +elif m == "fork_undeclared": + acknowledged(); second(); out["r"] = head() +elif m == "fork_shared": + e = acknowledged(); h = c.intake.row(e)["revision_hash"] + supersede(e, "branch one", h); supersede(e, "branch two", h); out["r"] = head() +elif m == "unknown_pred": + e = acknowledged(); supersede(e, "claims to replace something we never saw", "f" * 64); out["r"] = head() +elif m == "cycle": + e = acknowledged(); s = supersede(e, "second revision", c.intake.row(e)["revision_hash"]) + with c.store.transaction() as db: + db.execute("UPDATE revision_lineage SET supersedes_hash = ? WHERE event_id = ?", (s["revisionHash"], e)) + out["r"] = head() +elif m == "reversed": + acknowledged(text="a different revision"); second(text="the deliverable"); out["r"] = head() +elif m == "self": + rel = c.register(); c._rid = rel["relationshipId"] + p = c.ready_payload(rel, [c.artifact("out.txt", "self referential")]) + out["r"] = refusal(c.intake.accept_child_receipt, p, observation=c.assigned_turn(), supersedes_revision=p["revisionHash"]) +elif m == "managed_none": + e = acknowledged(); crit.set_mode(c._rid, "managed"); out["r"] = verdict(e, "verified") +elif m == "legacy": + e = acknowledged(); out["r"] = verdict(e, "verified") +else: + e = acknowledged() + if m == "no_claim" or m == "explicit": + reg = register_criteria() + else: + register_criteria(); c.ack.claim_verification(e, turn_id="ack-turn") + both = [{"id": "c1", "verdict": "verified"}, {"id": "c2", "verdict": "verified"}] + if m == "missing": out["r"] = verdict(e, "verified", findings=[{"id": "c1", "verdict": "verified"}]) + elif m == "covered": out["r"] = verdict(e, "verified", findings=both) + elif m == "no_note": out["r"] = verdict(e, "needs_changes", findings=[{"id": "c1", "verdict": "needs_changes"}]) + elif m == "unknown_id": out["r"] = verdict(e, "verified", findings=[{"id": "nope", "verdict": "verified"}]) + elif m == "bad_disposition": out["r"] = verdict(e, "verified", findings=[{"id": "c1", "verdict": "regressed"}]) + elif m == "edited": + register_criteria([{"id": "c1", "title": "the endpoint returns a COMPLETELY different shape"}, {"id": "c2", "title": "a malformed request is refused"}]) + out["r"] = verdict(e, "verified", findings=both) + elif m == "wrong_digest": out["r"] = verdict(e, "verified", findings=both, expect_criteria_digest="0" * 64) + elif m == "no_claim": out["r"] = verdict(e, "verified", findings=both) + elif m == "explicit": out["r"] = verdict(e, "verified", findings=both, expect_criteria_digest=reg["setDigest"]) diff --git a/internal/relay/delivery/testdata/scenarios/vcu_ack.py b/internal/relay/delivery/testdata/scenarios/vcu_ack.py new file mode 100644 index 00000000..f3eb818f --- /dev/null +++ b/internal/relay/delivery/testdata/scenarios/vcu_ack.py @@ -0,0 +1,37 @@ +from codex_session_relay import identity +def dispatched(): + _r, e = c.queued_event(recipients=[PARENT, CHILD]); c.attempt(e); c.clock.advance(5); return e +def advance(): + return c.registry.open_generation(c._rid, dispatch_request_id="newer-execution", reason="needs_changes_revision", dispatch_turn_id="newer-turn") +m = sys.argv[3] +e = dispatched() +if m in ("offline", "no_verdict"): + out["ack"] = c.ack.acknowledge(e, ack_turn_id="parent-own-turn", ack_proof=identity.ack_proof(e, "parent-own-turn"), accepted=True, adapter=None) + if m == "no_verdict": + out["r"] = refusal(c.ack.record_verdict, e, verdict="verified", verdict_turn_id="v1") +elif m == "dispatch_turn": + t = c.delivery.find(e)["dispatch_turn_id"] + out["ack"] = c.ack.acknowledge(e, ack_turn_id=t, ack_proof=identity.ack_proof(e, t), accepted=True, adapter=None) +else: + c.adapter.start_turn(PARENT, turn_id="parent-own-turn", status="inProgress") + out["ack"] = c.ack.acknowledge(e, ack_turn_id="parent-own-turn", ack_proof=identity.ack_proof(e, "parent-own-turn"), accepted=True, adapter=None) + if m == "upgrade": + out["r"] = c.ack.verify_pending_acks(c.adapter) + elif m == "advanced": + advance(); out["r"] = c.ack.verify_pending_acks(c.adapter) + elif m == "paused": + c.registry.set_status(c._rid, "paused", actor=PARENT); out["r"] = c.ack.verify_pending_acks(c.adapter) + elif m == "twice": + advance() + out["r"] = c.ack.verify_pending_acks(c.adapter, now=c.clock.now()) + out["r2"] = c.ack.verify_pending_acks(c.adapter, now=c.clock.now() + 10000) + elif m == "next_check": + advance() + out["r"] = c.ack.verify_pending_acks(c.adapter, now=100.0) + out["r2"] = c.ack.verify_pending_acks(c.adapter, now=101.0) + elif m == "cleared": + c.registry.set_status(c._rid, "paused", actor=PARENT) + out["r"] = c.ack.verify_pending_acks(c.adapter, now=100.0) + rel = c.registry.get(c._rid) + c.registry.resume(c._rid, expect_generation=rel["executionGeneration"], expect_artifact_roots=rel["authorizedScope"]["artifactRoots"], expect_allowed_recipients=rel["authorizedScope"]["allowedRecipients"], actor=PARENT) + out["r2"] = c.ack.verify_pending_acks(c.adapter, now=100000.0) diff --git a/internal/relay/delivery/transport.go b/internal/relay/delivery/transport.go new file mode 100644 index 00000000..f7f8a01b --- /dev/null +++ b/internal/relay/delivery/transport.go @@ -0,0 +1,193 @@ +package delivery + +import ( + "fmt" + "slices" + "strings" +) + +// Transport vocabulary (transport.py). These spellings are stored in the database and read by +// every status surface. +const ( + Accepted = "accepted" + FailedStatus = "failed" + OutcomeUnknown = "outcome_unknown" + Unfinished = "in_progress_or_unknown" + + Dispatched = "dispatched" + DeferredBusy = "deferred_busy" + WithheldPreSend = "withheld_pre_send" + HeldUncertain = "held_uncertain" + InboxOnly = "inbox_only" + Queued = "queued" + Acknowledged = "acknowledged" + Superseded = "superseded" + Sending = "sending" +) + +var knownMethods = []string{"initialize", "thread/read", "thread/resume", "turn/start"} + +// SettingsRefusals are the resume refusals decided before any turn/start (transport.py). +var SettingsRefusals = []string{"settings_not_preserved", "setting_unobservable", "environments_unknown", "unverifiable_permission_profile", "settings_differ_after_load"} + +// Facts is transport.TransportFacts: one receipt turned into delivery facts. +type Facts struct { + ReceiptStatus string + DeliveryState string + SendAttempted string + RetrySafe bool + FailedOperation any // string or nil + TurnID any // string or nil + ApprovalPolicy any + RPCErrorCode any + ErrorText any +} + +func methodPrefix(errorText any) (string, bool) { + text, ok := errorText.(string) + if !ok || !strings.Contains(text, ": ") { + return "", false + } + candidate, _, _ := strings.Cut(text, ": ") + return candidate, slices.Contains(knownMethods, candidate) +} + +func usableTurnID(v any) any { + if s, ok := v.(string); ok && strings.TrimSpace(s) != "" { + return s + } + return nil +} + +func truthy(v any) bool { + switch t := v.(type) { + case nil: + return false + case bool: + return t + case string: + return t != "" + case Obj: + return len(t) > 0 + case []any: + return len(t) > 0 + case int64: + return t != 0 + case float64: + return t != 0 + } + return true +} + +// Classify is transport.classify_operation_receipt. The error CODE is read before the method +// prefix, and an unfinished receipt is never proof of non-delivery. +func Classify(receipt Obj) Facts { + if receipt == nil { + return Facts{OutcomeUnknown, HeldUncertain, "unknown", false, "unclassified", nil, nil, nil, nil} + } + status, _ := get(receipt, "status") + errorText, _ := get(receipt, "error") + rpc, _ := get(receipt, "rpcError") + var code any + if o, ok := rpc.(Obj); ok { + code, _ = get(o, "code") + } + resumed, _ := get(receipt, "resumed") + var policy any + if o, ok := resumed.(Obj); ok { + policy, _ = get(o, "approvalPolicy") + } + rawTurn, _ := get(receipt, "turnId") + turn := usableTurnID(rawTurn) + f := func(rs, ds, sa string, safe bool, op any, turn any) Facts { + return Facts{rs, ds, sa, safe, op, turn, policy, code, errorText} + } + switch status { + case Accepted: + if turn != nil { + return f(Accepted, Dispatched, "yes", false, nil, turn) + } + return f(OutcomeUnknown, HeldUncertain, "unknown", false, "unclassified", nil) + case Unfinished: + return f(Unfinished, HeldUncertain, "unknown", false, nil, nil) + case OutcomeUnknown: + return f(OutcomeUnknown, HeldUncertain, "unknown", false, "transport", nil) + case FailedStatus: + if code == "thread_busy" && !truthy(resumed) { + return f(FailedStatus, DeferredBusy, "no", true, "thread/read", nil) + } + if code == "unsupported_approval_policy" && truthy(resumed) { + return f(FailedStatus, InboxOnly, "no", false, "thread/resume", nil) + } + if c, ok := code.(string); ok && slices.Contains(SettingsRefusals, c) && truthy(resumed) { + return f(FailedStatus, WithheldPreSend, "no", true, "thread/resume", nil) + } + prefix, known := methodPrefix(errorText) + switch { + case known && prefix == "initialize": + return f(FailedStatus, HeldUncertain, "unknown", false, "initialize", nil) + case known && prefix == "turn/start": + return f(FailedStatus, HeldUncertain, "yes", false, "turn/start", nil) + case known && (prefix == "thread/read" || prefix == "thread/resume") && !truthy(resumed): + return f(FailedStatus, WithheldPreSend, "no", true, prefix, nil) + } + } + return f(OutcomeUnknown, HeldUncertain, "unknown", false, "unclassified", nil) +} + +// AttemptRecord is transport.attempt_record: the frozen DeliveryAttempt shape. +func AttemptRecord(facts Facts, requestID, eventID string, attemptNo int64, recipient, statusBefore, observedAt string, reconciliation Obj) (Obj, error) { + record := Obj{ + {Key: "requestId", Value: requestID}, + {Key: "eventId", Value: eventID}, + {Key: "attemptNo", Value: attemptNo}, + {Key: "recipientTaskId", Value: recipient}, + {Key: "deliveryState", Value: facts.DeliveryState}, + {Key: "sendAttempted", Value: facts.SendAttempted}, + {Key: "retrySafe", Value: facts.RetrySafe}, + {Key: "observedAt", Value: observedAt}, + {Key: "recipientStatusBefore", Value: statusBefore}, + {Key: "recipientApprovalPolicy", Value: facts.ApprovalPolicy}, + {Key: "transportReceiptStatus", Value: facts.ReceiptStatus}, + {Key: "failedOperation", Value: facts.FailedOperation}, + {Key: "turnId", Value: facts.TurnID}, + } + if reconciliation != nil { + record = append(record, F{Key: "reconciliation", Value: reconciliation}) + } + return record, AssertAttemptInvariants(record) +} + +// AssertAttemptInvariants re-implements the frozen schema's conditional rules before persisting. +func AssertAttemptInvariants(record Obj) error { + state := str(record, "deliveryState") + safe, _ := get(record, "retrySafe") + sent := str(record, "sendAttempted") + status := str(record, "transportReceiptStatus") + op, _ := get(record, "failedOperation") + turn, _ := get(record, "turnId") + fail := func(rule string) error { return fmt.Errorf("delivery: attempt invariant: %s", rule) } + if state == HeldUncertain && (safe != false || (sent != "unknown" && sent != "yes")) { + return fail("an uncertain attempt is never retry-safe") + } + if safe == true { + if sent != "no" || status != FailedStatus || (state != WithheldPreSend && state != DeferredBusy) || (op != "thread/read" && op != "thread/resume") { + return fail("a retry-safe attempt is a completed pre-send failure") + } + } + if status == Unfinished && (sent != "unknown" || safe != false) { + return fail("an unfinished receipt is unknown and unsafe") + } + if state == Dispatched { + if t, ok := turn.(string); !ok || t == "" || sent != "yes" || safe != false { + return fail("a dispatch carries its turn") + } + } + switch op { + case "turn/start", "transport", "initialize", "unclassified": + if safe != false || (sent != "unknown" && sent != "yes") { + return fail("a post-send failure is never retry-safe") + } + } + return nil +} diff --git a/internal/relay/delivery/unknownsend_a_test.go b/internal/relay/delivery/unknownsend_a_test.go new file mode 100644 index 00000000..78435da7 --- /dev/null +++ b/internal/relay/delivery/unknownsend_a_test.go @@ -0,0 +1,637 @@ +package delivery + +import ( + "fmt" + "strings" + "testing" +) + +// test_unknown_send_lost.py USL-1..USL-11 (method: hostloss_harness_test.go). + +const usl = "test_unknown_send_lost" + +func uslMessage(request string) string { + return "[codex-session-relay] verification request\nrequestId: " + request +} + +// unknownSend is UnknownSendCase.unknown_send. +func (h *hl) unknownSend(history, restarted bool) (string, string) { + if history { + h.parentHistory() + } + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"transport_unknown"} + record := h.attemptOn(event, h.host, nil) + h.eq(field(record, "transportReceiptStatus")) + h.eq(field(record, "turnId")) + h.eq(h.row(event).S("state")) + if restarted { + h.host.threads[parent].status = "notLoaded" + } + return event, str(record, "requestId") +} + +func (h *hl) ticks(count int, seconds float64) { + for i := 0; i < count; i++ { + h.clock.Advance(seconds) + h.tick() + } +} + +func (h *hl) sendsTo(recipient string) []any { + out := []any{} + for _, s := range h.host.sends { + if s.thread == recipient { + out = append(out, s.requestID) + } + } + return out +} + +func (h *hl) mark(request string) any { + return h.one("SELECT recipient_scan FROM attempts WHERE request_id = ?", request).Opt("recipient_scan") +} + +// argvTail is shlex.split(command)[-5:] for the commands recoveryCommand renders. +func argvTail(command any) []any { + text, _ := command.(string) + var argv []string + for _, word := range strings.Fields(text) { + argv = append(argv, strings.Trim(word, "'")) + } + out := []any{} + for _, w := range argv[max(0, len(argv)-5):] { + out = append(out, w) + } + return out +} + +// assertHeld is UnknownSendCase.assert_held: the values each of its assertions reads. +func (h *hl) assertHeld(event, request string) { + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.eq([]any{row.Opt("dispatch_evidence"), row.Opt("dispatch_turn_id")}) + h.eq(h.mark(request)) + states := h.attemptStates(event) + h.eq(states[len(states)-1]) + h.eq(h.statusPair(event)) + h.eq(h.nextAction()) + h.eq(field(h.completionDelivery(), "turnCheck")) + recovery := sub(h.assignment(), "recovery") + h.eq([]any{field(recovery, "actor"), field(recovery, "reason")}) + h.eq(argvTail(field(recovery, "command"))) +} + +func Test21_USL01_an_unknown_send_without_trace_is_held_for_the_parent_by_name(t *testing.T) { + const cls = "AnUnknownSendTheHostKeptNoTraceOf." + t.Run("daemon tick", func(t *testing.T) { + mirror(t, usl, cls+"test_it_is_held_for_the_parent_by_name_and_never_sent_again", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.attemptStates(event)) + h.ticks(6, 700) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("recipient still held by its host", func(t *testing.T) { + mirror(t, usl, cls+"test_a_recipient_its_host_still_holds_is_held_the_same_way", func(h *hl) { + event, first := h.unknownSend(true, false) + h.ticks(3, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("reconcile twice", func(t *testing.T) { + mirror(t, usl, cls+"test_reconciling_it_again_keeps_the_hold_and_sends_nothing", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + again := h.reconcile(first, h.host) + h.eq([]any{field(again, "state"), field(again, "nextExpectedAction")}) + h.assertHeld(event, first) + h.eq(h.attemptStates(event)) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("an ACK without a trace", func(t *testing.T) { + mirror(t, usl, cls+"test_an_acknowledgement_without_a_trace_is_held_the_same_way", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(5) + turn := h.host.startTurn(parent, "ack-turn", "completed", "") + _, err := h.ack.Acknowledge(h.ctx, event, turn.TurnID, AckProof(event, turn.TurnID), true, nil, h.host) + mustDo(t, err) + h.eq(h.ackRow(event).Opt("last_reason")) + h.ticks(3, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + h.eq(h.faultIn("open", "broken")) + }) + }) +} + +func Test21_USL02_reconcile_names_the_parent_the_reason_and_the_command(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_reconcile_names_the_parent_the_reason_and_the_command_without_sending", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(120) + outcome := h.reconcile(first, h.host) + h.eq([]any{field(outcome, "state"), field(outcome, "evidence")}) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + h.eq(field(sub(outcome, "recipientTrace"), "finding")) + recovery := sub(outcome, "recovery") + h.eq(field(recovery, "actor")) + h.eq(argvTail(field(recovery, "command"))) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) +} + +// withReadUnknownSend replaces the hostloss.read_unknown_send seam for one call (mock.patch). +func withReadUnknownSend(replacement func(Adapter, Clock, Row, Row, string) Reading, body func()) { + original := readUnknownSend + readUnknownSend = replacement + defer func() { readUnknownSend = original }() + body() +} + +func Test21_USL03_a_hold_that_loses_its_race_to_a_confirmation_reports_the_confirmation(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_a_hold_that_loses_its_race_to_a_confirmation_reports_the_confirmation", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(120) + original := readUnknownSend + var outcome Obj + withReadUnknownSend(func(a Adapter, c Clock, attempt, delivery Row, receipt string) Reading { + reading := original(a, c, attempt, delivery, receipt) + h.host.startTurn(parent, "", "completed", uslMessage(first)) + out, err := h.rc.ReconcileAttempt(h.ctx, first, h.host, nil) + mustDo(h.t, err) + h.eq(field(out, "evidence")) + return reading + }, func() { outcome = h.reconcile(first, h.host) }) + h.eq([]any{field(outcome, "deliveryState"), field(outcome, "evidence")}) + _, has := get(outcome, "nextExpectedAction") + h.eq(has) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.eq(h.nextAction()) + h.eq(h.sendsTo(parent)) + }) +} + +func Test21_USL04_no_hold_is_named_while_the_daemon_can_still_decide(t *testing.T) { + const cls = "AnUnknownSendTheHostKeptNoTraceOf." + t.Run("within the allowance", func(t *testing.T) { + mirror(t, usl, cls+"test_a_send_too_recent_to_judge_is_read_again_until_it_is_held", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 20) + h.eq(h.attemptStates(event)) + h.eq(h.row(event).Opt("hold_reason")) + h.eq(h.nextAction()) + h.ticks(1, 60) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("a turn running since the send", func(t *testing.T) { + mirror(t, usl, cls+"test_a_turn_still_running_since_the_send_holds_the_decision_until_it_ends", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(5) + running := h.host.startTurn(parent, "", "inProgress", "") + h.ticks(1, 120) + h.eq(h.attemptStates(event)) + h.eq(h.row(event).Opt("hold_reason")) + h.eq(h.nextAction()) + h.host.finishTurn(parent, running.TurnID, "completed") + h.ticks(1, 30) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("an older turn still running", func(t *testing.T) { + mirror(t, usl, cls+"test_an_older_turn_still_running_holds_the_decision", func(h *hl) { + event, first := h.foldedUnknownSend(205, true, "") + t := h.host.threads[parent] + kept := t.items[:0:0] + for _, item := range t.items { + if !strings.Contains(item[1], first) { + kept = append(kept, item) + } + } + t.items = kept + h.ticks(1, 120) + h.eq(h.attemptStates(event)) + h.eq(h.row(event).Opt("hold_reason")) + h.eq(h.nextAction()) + h.eq(h.sendsTo(parent)) + }) + }) +} + +// foldedUnknownSend is UnknownSendCase.folded_unknown_send; kind "" is the message itself. +func (h *hl) foldedUnknownSend(later int, running bool, kind string) (string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.startTurn(parent, "folded", "inProgress", "") + h.clock.Advance(120) + h.host.script = []string{"transport_unknown"} + first := str(h.attemptOn(event, h.host, nil), "requestId") + if kind == "" { + h.item(parent, "folded", uslMessage(first), "") + } else { + h.item(parent, "folded", "hook saw "+first, kind) + } + for n := 0; n < later; n++ { + h.item(parent, "folded", fmt.Sprintf("later work %d", n), "commandExecution") + } + if !running { + h.host.finishTurn(parent, "folded", "completed") + } + return event, first +} + +func Test21_USL05_a_message_that_turns_up_later_confirms_the_send_and_clears_the_hold(t *testing.T) { + t.Run("after a lost hold", func(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_a_message_that_turns_up_after_the_hold_confirms_the_send_and_clears_it", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.host.startTurn(parent, "", "completed", uslMessage(first)) + h.ticks(1, 120) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.eq(h.evidenceOf(event)) + h.eq(h.nextAction()) + _, has := get(h.assignment(), "recovery") + h.eq(has) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("after an undecided hold", func(t *testing.T) { + mirror(t, usl, "AnUndecidedReadingIsHeldByName.test_a_message_found_later_clears_the_undecided_hold", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(2) + hook := h.host.startTurn(parent, "", "completed", "") + h.item(parent, hook.TurnID, "hook saw "+first, "hookPrompt") + h.ticks(1, 120) + h.eq(h.row(event).Opt("hold_reason")) + h.item(parent, hook.TurnID, uslMessage(first), "") + h.ticks(1, 120) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.eq(h.nextAction()) + }) + }) + t.Run("between the two scans", func(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_a_message_that_lands_between_the_two_scans_confirms_the_send", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(3) + late := h.host.startTurn(parent, "", "completed", "") + h.clock.Advance(120) + lands := &hooked{Adapter: h.host} + lands.findToken = func(thread, token string, limit int, messageOnly bool) (TokenScan, error) { + scan, err := h.host.FindToken(thread, token, limit, messageOnly) + h.item(parent, late.TurnID, uslMessage(first), "") + return scan, err + } + outcome := h.reconcile(first, lands) + h.eq(field(outcome, "evidence")) + h.eq(h.row(event).S("state")) + h.eq(h.attemptStates(event)) + h.eq(h.sendsTo(parent)) + }) + }) +} + +func Test21_USL06_an_unknown_send_after_a_host_loss_is_held_and_claims_no_dispatch(t *testing.T) { + t.Run("lost: dispatch cleared", func(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_an_unknown_send_after_a_host_lost_turn_is_held_and_claims_no_dispatch", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.host.script = []string{"transport_unknown"} + h.ticks(1, 120) + h.eq(h.attemptStates(event)) + second := h.attemptsFor(event)[1].S("request_id") + h.host.threads[parent].status = "notLoaded" + h.ticks(3, 120) + h.eq(h.attemptStates(event)) + h.assertHeld(event, second) + h.eq(h.sendsTo(parent)) + _ = first + }) + }) + t.Run("undecided: dispatch cleared", func(t *testing.T) { + mirror(t, usl, "AnUndecidedReadingIsHeldByName.test_an_undecided_redelivery_after_a_host_loss_keeps_no_dispatch_evidence", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + row := h.row(event) + h.eq([]any{row.Opt("dispatch_evidence"), row.Opt("dispatch_turn_id")}) + claimed, err := h.delivery.claim(h.ctx, event, h.clock.Now(), "relay", parent) + mustDo(t, err) + request := claimed.requestID + h.host.ledger[request] = Obj{{Key: "requestId", Value: request}, {Key: "operation", Value: "send_message_to_thread"}, {Key: "status", Value: "outcome_unknown"}, + {Key: "threadId", Value: parent}, {Key: "retrySafe", Value: false}, {Key: "error", Value: "TransportError: turn/start: response unavailable; do not resend"}} + h.clock.Advance(2) + hook := h.host.startTurn(parent, "", "completed", "") + h.item(parent, hook.TurnID, "hook saw "+request, "hookPrompt") + h.clock.Advance(120) + h.reconcile(request, h.host) + row = h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.eq([]any{row.Opt("dispatch_evidence"), row.Opt("dispatch_turn_id")}) + }) + }) +} + +func Test21_USL07_a_send_folded_into_an_older_turn_is_found_there(t *testing.T) { + const cls = "AnUnknownSendTheHostKeptNoTraceOf." + t.Run("folded into an older turn", func(t *testing.T) { + mirror(t, usl, cls+"test_a_send_folded_into_an_older_turn_is_found_there_and_not_sent_again", func(h *hl) { + event, first := h.foldedUnknownSend(205, false, "") + h.ticks(1, 120) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + h.ticks(3, 120) + h.eq(h.attemptStates(event)) + h.eq(h.sendsTo(parent)) + _ = first + }) + }) + t.Run("a turn begun just after the send", func(t *testing.T) { + mirror(t, usl, cls+"test_a_turn_begun_just_after_the_send_does_not_hide_the_folded_one", func(h *hl) { + event, _ := h.foldedUnknownSend(205, false, "") + h.clock.Advance(0.5) + h.host.startTurn(parent, "later", "completed", "unrelated") + h.ticks(1, 120) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("a turn begun just before the send", func(t *testing.T) { + mirror(t, usl, cls+"test_a_send_folded_into_a_turn_begun_just_before_it_is_found_there", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.startTurn(parent, "just-before", "inProgress", "") + h.clock.Advance(10) + h.host.script = []string{"transport_unknown"} + first := str(h.attemptOn(event, h.host, nil), "requestId") + h.item(parent, "just-before", uslMessage(first), "") + for n := 0; n < 205; n++ { + h.item(parent, "just-before", fmt.Sprintf("later work %d", n), "commandExecution") + } + h.host.finishTurn(parent, "just-before", "completed") + h.host.threads[parent].status = "notLoaded" + h.ticks(1, 120) + h.eq(h.evidenceOf(event)) + h.eq(h.row(event).S("state")) + h.eq(h.sendsTo(parent)) + }) + }) +} + +func Test21_USL08_undecided_readings_are_held_for_the_parent_by_name(t *testing.T) { + heldFolded := func(name string, later func(h *hl)) { + t.Run(name, func(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf."+name, func(h *hl) { + event, first := h.foldedUnknownSend(205, false, "hookPrompt") + later(h) + h.host.threads[parent].status = "notLoaded" + h.ticks(1, 120) + h.assertHeld(event, first) + if strings.Contains(name, "of_the_folded_turn") { + h.ticks(3, 700) + } + h.eq(h.sendsTo(parent)) + }) + }) + } + heldFolded("test_a_token_only_in_a_hook_prompt_of_the_folded_turn_is_held_by_name", func(*hl) {}) + heldFolded("test_a_hook_prompt_in_the_folded_turn_behind_a_later_turn_is_held_by_name", func(h *hl) { + h.clock.Advance(0.5) + h.host.startTurn(parent, "later", "completed", "unrelated") + }) + t.Run("2000+205 items", func(t *testing.T) { + mirror(t, usl, "AnUnknownSendTheHostKeptNoTraceOf.test_an_older_turn_too_long_to_read_through_holds_the_send_by_name", func(h *hl) { + event, first := h.foldedUnknownSend(0, false, "") + t := h.host.threads[parent] + carried := t.items[len(t.items)-1] + t.items = t.items[:len(t.items)-1] + for n := 0; n < 2000; n++ { + h.item(parent, "folded", fmt.Sprintf("earlier work %d", n), "commandExecution") + } + t.items = append(t.items, carried) + for n := 0; n < 205; n++ { + h.item(parent, "folded", fmt.Sprintf("later work %d", n), "commandExecution") + } + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + const cls = "AnUndecidedReadingIsHeldByName." + t.Run("empty listing", func(t *testing.T) { + mirror(t, usl, cls+"test_a_parent_listing_no_turns_holds_the_send_by_name", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.ticks(3, 700) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("another item type", func(t *testing.T) { + mirror(t, usl, cls+"test_a_token_only_in_another_item_type_is_held_by_name", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(2) + hook := h.host.startTurn(parent, "", "completed", "") + h.item(parent, hook.TurnID, "hook saw "+first, "hookPrompt") + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("scan bounded before the send", func(t *testing.T) { + mirror(t, usl, cls+"test_a_scan_bounded_before_the_send_is_held_by_name", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(2) + busy := h.host.startTurn(parent, "", "completed", "") + for n := 0; n < 5; n++ { + h.item(parent, busy.TurnID, fmt.Sprintf("work %d", n), "commandExecution") + } + h.host.scanLimit = 3 + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("listing never reaches the send", func(t *testing.T) { + mirror(t, usl, cls+"test_a_listing_that_never_reaches_the_send_is_held_by_name", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(120) + outcome := h.reconcile(first, h.neverReachesTheSend()) + h.eq(field(outcome, "nextExpectedAction")) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) +} + +func (h *hl) neverReachesTheSend() *hooked { + return &hooked{Adapter: h.host, findDispatched: func(string, string, float64) (TurnPresence, error) { + return TurnPresence{}, &ListingBounded{"the bounded listing never reached the send"} + }} +} + +func Test21_USL09_a_listed_turn_without_an_id_is_not_taken_for_the_sends_turn(t *testing.T) { + mirror(t, usl, "TheListingSinceASendWithNoTurnId.test_a_listed_turn_without_an_id_is_not_taken_for_the_sends_turn", func(h *hl) { + hundred, ten := 100.0, 10.0 + presence, err := FindInListing([]ListingPage{{Turns: []TurnInfo{{TurnID: "", Status: "completed", StartedAt: &hundred}, {TurnID: "older", Status: "completed", StartedAt: &ten}}}}, "", 150.0) + mustDo(t, err) + h.eq([]any{presence.Finding, presence.Stop, presence.Seen, strs(presence.Older)}) + statuses := []any{} + for _, turn := range presence.SeenTurns { + statuses = append(statuses, turn.Status) + } + h.eq(statuses) + h.eq(presence.StopTurn.TurnID) + }) +} + +func Test21_USL10_a_send_the_transport_has_not_answered_is_held_by_name(t *testing.T) { + const cls = "ASendTheTransportHasNotAnsweredIsHeldByName." + unfinished := func(h *hl) (string, string) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + h.host.script = []string{"in_progress"} + request := str(h.attemptOn(event, h.host, nil), "requestId") + h.host.threads[parent].status = "notLoaded" + return event, request + } + t.Run("in_progress receipt", func(t *testing.T) { + mirror(t, usl, cls+"test_an_unfinished_receipt_is_held_for_the_parent_by_name", func(h *hl) { + event, first := unfinished(h) + h.clock.Advance(120) + outcome := h.reconcile(first, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + h.ticks(3, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("settles later", func(t *testing.T) { + mirror(t, usl, cls+"test_a_receipt_that_settles_later_lets_the_rule_decide", func(h *hl) { + event, first := unfinished(h) + h.ticks(1, 120) + h.eq(h.row(event).Opt("hold_reason")) + receipt := append(Obj(nil), h.host.ledger[first]...) + receipt = set(receipt, "status", "outcome_unknown") + receipt = set(receipt, "error", "TransportError: turn/start: no answer") + h.host.ledger[first] = receipt + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("no receipt", func(t *testing.T) { + mirror(t, usl, cls+"test_a_claim_with_no_receipt_is_held_for_the_parent_by_name", func(h *hl) { + h.parentHistory() + event := h.queuedEvent(regOpts{}) + claimed, err := h.delivery.claim(h.ctx, event, h.clock.Now(), "relay", parent) + mustDo(t, err) + h.clock.Advance(120) + outcome := h.reconcile(claimed.requestID, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + h.eq(h.attemptStates(event)) + h.eq(sendsJSON(h.host)) + }) + }) + t.Run("unreadable receipt", func(t *testing.T) { + mirror(t, usl, cls+"test_an_unreadable_receipt_takes_no_reading", func(h *hl) { + event, first := unfinished(h) + h.clock.Advance(120) + h.host.readFailures["get_operation"] = true + outcome := h.reconcile(first, h.host) + _, has := get(outcome, "recipientTrace") + h.eq(has) + h.eq(field(outcome, "nextExpectedAction")) + h.eq(h.row(event).Opt("hold_reason")) + }) + }) +} + +func Test21_USL11_a_deciding_reading_replaces_an_undecided_hold_and_nothing_else_does(t *testing.T) { + const cls = "AnUndecidedReadingIsHeldByName." + t.Run("a later deciding reading", func(t *testing.T) { + mirror(t, usl, cls+"test_a_later_reading_that_decides_replaces_the_undecided_hold", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.host.startTurn(parent, "", "completed", "the operator asked something") + h.ticks(1, 120) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("a pass that cannot decide", func(t *testing.T) { + mirror(t, usl, cls+"test_a_pass_that_cannot_decide_keeps_the_undecided_hold", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.host.startTurn(parent, "", "inProgress", "") + h.clock.Advance(20) + pending := h.reconcile(first, h.host) + h.eq(field(sub(pending, "recipientTrace"), "pending")) + h.eq(field(pending, "nextExpectedAction")) + h.assertHeld(event, first) + h.host.readFailures["find_token"] = true + unread := h.reconcile(first, h.host) + _, has := get(unread, "recipientTrace") + h.eq(has) + h.eq([]any{field(unread, "nextExpectedAction"), field(unread, "reason")}) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("an older reading written last", func(t *testing.T) { + mirror(t, usl, cls+"test_an_older_reading_does_not_overwrite_a_newer_hold", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + original := readUnknownSend + var raced []Reading + var outcome Obj + withReadUnknownSend(func(a Adapter, c Clock, attempt, delivery Row, receipt string) Reading { + reading := original(a, c, attempt, delivery, receipt) + if len(raced) == 0 { + raced = append(raced, reading) + h.host.startTurn(parent, "", "completed", "the operator asked") + newer, err := h.rc.ReconcileAttempt(h.ctx, first, h.host, nil) + mustDo(h.t, err) + h.eq(field(newer, "reason")) + } + return reading + }, func() { outcome = h.reconcile(first, h.host) }) + h.eq(field(raced[0], "undecided")) + h.eq(truthy(field(outcome, "changed"))) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + h.eq(argvTail(field(sub(outcome, "recovery"), "command"))) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("re-read though the items did not change", func(t *testing.T) { + mirror(t, usl, cls+"test_an_undecided_hold_is_read_again_later_though_the_items_did_not_change", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.host.startTurn(parent, "", "completed", "") + h.ticks(1, 30) + h.assertHeld(event, first) + h.ticks(1, 600) + h.assertHeld(event, first) + h.eq(h.sendsTo(parent)) + }) + }) +} diff --git a/internal/relay/delivery/unknownsend_b_test.go b/internal/relay/delivery/unknownsend_b_test.go new file mode 100644 index 00000000..da4fabd8 --- /dev/null +++ b/internal/relay/delivery/unknownsend_b_test.go @@ -0,0 +1,598 @@ +package delivery + +import ( + "context" + "database/sql" + "math" + "regexp" + "sort" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/faults" +) + +// test_unknown_send_lost.py USL-12..USL-22 (method: hostloss_harness_test.go). USL-18 is +// python-internal (cli.Services wiring by identity) and not ported. + +func Test21_USL12_a_persisting_hold_opens_its_fault_as_broken(t *testing.T) { + const cls = "APersistingHoldReachesTheOperator." + t.Run("lost", func(t *testing.T) { + mirror(t, usl, cls+"test_a_lost_hold_opens_its_fault_instead_of_staying_observed", func(h *hl) { + h.unknownSend(true, true) + h.ticks(1, 120) + h.eq(h.faultIn("open", "broken")) + }) + }) + t.Run("undecided", func(t *testing.T) { + mirror(t, usl, cls+"test_an_undecided_hold_opens_its_fault_instead_of_staying_observed", func(h *hl) { + h.unknownSend(false, true) + h.ticks(1, 120) + h.eq(h.faultIn("open", "broken")) + }) + }) + t.Run("within the allowance", func(t *testing.T) { + mirror(t, usl, cls+"test_a_wait_the_daemon_still_ends_stays_below_the_threshold", func(h *hl) { + h.unknownSend(true, true) + h.ticks(1, 20) + h.eq(h.faultIn("open", "broken")) + }) + }) +} + +func Test21_USL13_a_revision_request_left_without_trace_is_held_not_resent(t *testing.T) { + mirror(t, usl, "ACorrectionLostTheSameWayIsTheParentsToRecover.test_a_revision_request_left_without_trace_is_held_not_resent", func(h *hl) { + _, correction := h.correctionAfterNeedsChanges() + h.host.startTurn(child, "child-earlier", "completed", "") + h.clock.Advance(300) + h.host.script = []string{"transport_unknown"} + record := h.attemptOn(correction, h.host, nil) + h.eq(h.row(correction).S("state")) + h.host.threads[child].status = "notLoaded" + h.clock.Advance(120) + outcome := h.reconcile(str(record, "requestId"), h.host) + h.eq(field(outcome, "nextExpectedAction")) + h.eq(argvTail(field(sub(outcome, "recovery"), "command"))) + row := h.row(correction) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.ticks(3, 700) + h.eq(h.sendsTo(child)) + h.eq(h.nextAction()) + h.eq(argvTail(field(sub(h.assignment(), "recovery"), "command"))) + }) +} + +func (h *hl) reading(event string) []any { + return []any{h.attemptStates(event), field(h.statusOf(event), "phase"), h.nextAction()} +} + +// fillWindow is UnknownSendCase.fill_window. +func (h *hl) fillWindow(now float64) float64 { + window := math.Floor(now/3600) * 3600 + h.exec("INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,?,?)", parent, int64(window), h.delivery.Policy.MaxSendsPerRecipientPerHour, now-600) + return window +} + +func Test21_USL14_the_four_K5_cases_read_apart(t *testing.T) { + const cls = "TheFourCasesReadApart." + t.Run("death before the answer, turn lost", func(t *testing.T) { + mirror(t, usl, cls+"test_death_before_the_answer_with_the_turn_lost", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.eq(h.reading(event)) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("death before the answer, turn kept", func(t *testing.T) { + mirror(t, usl, cls+"test_death_before_the_answer_with_the_turn_run_and_kept", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(3) + h.host.startTurn(parent, "", "completed", uslMessage(first)) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.eq(h.reading(event)) + h.eq(h.evidenceOf(event)) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("death after the answer, accepted turn lost", func(t *testing.T) { + mirror(t, usl, cls+"test_death_after_the_answer_with_the_accepted_turn_lost", func(h *hl) { + event, first, turn := h.dispatched() + h.hostLoses(turn, true) + h.clock.Advance(120) + h.reconcile(first, h.host) + h.eq(h.reading(event)) + row := h.row(event) + h.eq([]any{row.S("state"), row.Opt("hold_reason"), row.Opt("dispatch_evidence")}) + }) + }) + t.Run("the hourly cap", func(t *testing.T) { + mirror(t, usl, cls+"test_a_delivery_waiting_on_the_hourly_cap", func(h *hl) { + event := h.queuedEvent(regOpts{}) + now := h.clock.Now() + h.fillWindow(now) + h.eq(h.attemptOn(event, h.host, &now)) + h.eq(h.reading(event)) + h.eq(field(sub(h.statusOf(event), "pacing"), "reopensAt")) + h.eq(sendsJSON(h.host)) + }) + }) +} + +func (h *hl) capped() (string, float64, float64) { + event := h.queuedEvent(regOpts{}) + now := h.clock.Now() + return event, now, h.fillWindow(now) +} + +func pacingOf(item Obj) Obj { return sub(item, "pacing") } + +func Test21_USL15_the_hourly_cap_is_named_with_its_reopen_time(t *testing.T) { + const cls = "TheHourlyCapIsNamedWithItsReopenTime." + t.Run("status names the cap and a reopen time that does not move", func(t *testing.T) { + mirror(t, usl, cls+"test_status_names_the_cap_and_a_reopen_time_that_does_not_move", func(h *hl) { + event, now, _ := h.capped() + h.eq(h.attemptOn(event, h.host, &now)) + item := h.statusOf(event) + p := pacingOf(item) + h.eq([]any{field(p, "reason"), field(p, "reopensAt")}) + h.eq([]any{field(p, "sends"), field(p, "cap")}) + h.eq(field(item, "nextEligibleAt")) + h.eq(field(item, "holdReason")) + h.clock.Advance(30) + h.eq(h.attemptOn(event, h.host, at(h.clock.Now()))) + h.eq(field(pacingOf(h.statusOf(event)), "reopensAt")) + h.clock.Advance(60) + h.eq(h.attemptOn(event, h.host, at(h.clock.Now()))) + h.eq(field(pacingOf(h.statusOf(event)), "reopensAt")) + h.eq(sendsJSON(h.host)) + }) + }) + t.Run("assignment-show names the cap", func(t *testing.T) { + mirror(t, usl, cls+"test_assignment_show_names_the_cap_and_its_reopen_time", func(h *hl) { + event, now, _ := h.capped() + h.eq(h.attemptOn(event, h.host, &now)) + p := sub(h.completionDelivery(), "pacing") + h.eq([]any{field(p, "reason"), field(p, "reopensAt")}) + h.eq(h.nextAction()) + }) + }) + t.Run("a claim refused on the cap reads the budget again within a minute", func(t *testing.T) { + mirror(t, usl, cls+"test_a_claim_refused_on_the_cap_reads_the_budget_again_within_a_minute", func(h *hl) { + event, now, _ := h.capped() + h.delivery.RateLimited = func(string, float64) bool { return false } + h.eq(h.attemptOn(event, h.host, &now)) + h.eq(h.row(event).Opt("next_eligible_at")) + }) + }) + t.Run("a spent cap is named before the gap", func(t *testing.T) { + mirror(t, usl, cls+"test_a_spent_cap_is_named_before_the_gap_when_both_hold", func(h *hl) { + event := h.queuedEvent(regOpts{}) + now := h.clock.Now() + window := math.Floor(now/3600) * 3600 + h.exec("INSERT INTO recipient_rate (recipient_task_id, window_start, sends, last_send_at) VALUES (?,?,?,?)", parent, int64(window), h.delivery.Policy.MaxSendsPerRecipientPerHour, now-1) + h.eq(h.attemptOn(event, h.host, &now)) + item := h.statusOf(event) + p := pacingOf(item) + h.eq([]any{field(p, "reason"), field(p, "reopensAt"), field(item, "phase"), field(item, "nextEligibleAt")}) + }) + }) + t.Run("the delivery goes out when the window reopens", func(t *testing.T) { + mirror(t, usl, cls+"test_the_delivery_goes_out_when_the_window_reopens", func(h *hl) { + event, now, window := h.capped() + h.eq(h.attemptOn(event, h.host, &now)) + h.clock.Advance(window + 3600 - now) + record := h.attemptOn(event, h.host, nil) + h.eq(field(record, "deliveryState")) + h.eq(field(h.statusOf(event), "pacing")) + }) + }) +} + +func Test21_USL16_a_raised_or_lifted_cap_releases_the_delivery_within_a_minute(t *testing.T) { + const cls = "TheHourlyCapIsNamedWithItsReopenTime." + t.Run("cap raised", func(t *testing.T) { + mirror(t, usl, cls+"test_a_raised_cap_releases_the_delivery_within_a_minute", func(h *hl) { + event, now, window := h.capped() + h.eq(h.attemptOn(event, h.host, &now)) + raised := NewService(h.store, h.clock) + raised.Policy.MaxSendsPerRecipientPerHour = 24 + h.clock.Advance(60) + h.eq(h.clock.Now() < window+3600) + record, err := raised.Attempt(h.ctx, event, h.host, at(h.clock.Now()), "") + mustDo(t, err) + h.eq(field(record, "deliveryState")) + }) + }) + t.Run("zero cap lifted", func(t *testing.T) { + mirror(t, usl, cls+"test_a_lifted_zero_cap_releases_the_delivery_within_a_minute", func(h *hl) { + zero := NewService(h.store, h.clock) + zero.Policy.MaxSendsPerRecipientPerHour = 0 + event := h.queuedEvent(regOpts{}) + now := h.clock.Now() + record, err := zero.Attempt(h.ctx, event, h.host, &now, "") + mustDo(t, err) + h.eq(record) + h.clock.Advance(60) + h.eq(field(h.attemptOn(event, h.host, at(h.clock.Now())), "deliveryState")) + }) + }) +} + +func Test21_USL17_a_cap_of_zero_names_the_operator(t *testing.T) { + const cls = "TheHourlyCapIsNamedWithItsReopenTime." + t.Run("pacing says only a changed policy reopens it", func(t *testing.T) { + mirror(t, usl, cls+"test_a_cap_of_zero_says_nothing_reopens_it_but_a_changed_policy", func(h *hl) { + policy := DefaultPolicy() + policy.MaxSendsPerRecipientPerHour = 0 + p := policy.Pacing(h.clock.Now(), 0, nil) + h.eq([]any{field(p, "reason"), field(p, "reopensAt")}) + h.eq(regexp.MustCompile("changed policy").MatchString(str(p, "detail"))) + }) + }) + t.Run("completion", func(t *testing.T) { + mirror(t, usl, cls+"test_a_cap_of_zero_is_read_again_each_minute_and_names_the_operator", func(h *hl) { + zero := NewService(h.store, h.clock) + zero.Policy.MaxSendsPerRecipientPerHour = 0 + event := h.queuedEvent(regOpts{}) + now := h.clock.Now() + record, err := zero.Attempt(h.ctx, event, h.host, &now, "") + mustDo(t, err) + h.eq(record) + h.eq(h.row(event).Opt("next_eligible_at")) + h.delivery = zero + h.eq(h.nextAction()) + h.eq(sendsJSON(h.host)) + }) + }) + t.Run("correction", func(t *testing.T) { + mirror(t, usl, cls+"test_a_zero_cap_names_the_operator_for_a_correction", func(h *hl) { + h.correctionAfterNeedsChanges() + h.delivery.Policy.MaxSendsPerRecipientPerHour = 0 + h.eq(h.nextAction()) + }) + }) +} + +func Test21_USL19_a_delivery_held_by_its_own_later_backoff_is_not_paced(t *testing.T) { + mirror(t, usl, "TheHourlyCapIsNamedWithItsReopenTime.test_a_delivery_held_by_its_own_later_backoff_is_not_reported_as_paced", func(h *hl) { + event, _, window := h.capped() + later := window + 3600 + 1800 + h.exec("UPDATE deliveries SET state = 'deferred_busy', next_eligible_at = ? WHERE event_id = ?", later, event) + item := h.statusOf(event) + h.eq([]any{field(item, "pacing"), field(item, "phase"), field(item, "nextEligibleAt")}) + h.eq(field(h.completionDelivery(), "pacing")) + }) +} + +// ---------------------------------------------------------------- USL-20, USL-21: the fault + +func (h *hl) stall() Row { + return h.one("SELECT * FROM fault_ledger WHERE fault_class = 'delivery_stalled'") +} + +var namingSeq = regexp.MustCompile(`:held:(\w+):\d+$`) + +func (h *hl) keys() []any { + rows, err := all(h.ctx, h.store, "SELECT o.occurrence_key FROM fault_occurrences o JOIN fault_ledger f ON f.fault_id = o.fault_id WHERE f.fault_class = 'delivery_stalled'") + mustDo(h.t, err) + var keys []string + for _, r := range rows { + keys = append(keys, namingSeq.ReplaceAllString(r.S("occurrence_key"), ":held:$1:#")) + } + sort.Strings(keys) + return anySlice(keys) +} + +func (h *hl) publications() []any { + rows, err := all(h.ctx, h.store, "SELECT p.trigger_key, p.summary FROM fault_publications p JOIN fault_ledger f ON f.fault_id = p.fault_id WHERE f.fault_class = 'delivery_stalled' ORDER BY p.rowid") + mustDo(h.t, err) + out := []any{} + for _, r := range rows { + out = append(out, []any{r.S("trigger_key"), r.S("summary")}) + } + return out +} + +// assertBrokenNaming is assert_broken_naming: the fault states, then "is held: " in the +// stall's detail. +func (h *hl) assertBrokenNaming(hold string) { + h.eq(h.faultStates()) + h.eq(regexp.MustCompile(regexp.QuoteMeta("is held: " + hold)).MatchString(h.stall().S("detail"))) +} + +func publishedWith(pubs []any, trigger string) []string { + var out []string + for _, p := range pubs { + pair := p.([]any) + if pair[0] == trigger { + out = append(out, pair[1].(string)) + } + } + return out +} + +func Test21_USL20_a_hold_reaches_its_fault_whatever_the_sweep_saw_first(t *testing.T) { + const cls = "AHoldReachesItsFaultWhateverTheSweepSawFirst." + t.Run("lost after the sweep", func(t *testing.T) { + mirror(t, usl, cls+"test_a_hold_named_after_the_sweep_recorded_its_attempt_breaks_the_fault", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(20) + h.eq(h.faultStates()) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.eq(h.keys()) + h.eq(h.stall().I("occurrence_count")) + opened := publishedWith(h.publications(), "open") + h.eq(len(opened) > 0) + h.eq(regexp.MustCompile("severity: broken").MatchString(opened[len(opened)-1])) + h.eq(regexp.MustCompile("is held: " + UnknownSendLost).MatchString(opened[len(opened)-1])) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("undecided after the sweep", func(t *testing.T) { + mirror(t, usl, cls+"test_an_undecided_hold_named_after_the_sweep_breaks_the_fault_too", func(h *hl) { + event, first := h.unknownSend(false, true) + h.clock.Advance(20) + h.eq(h.faultStates()) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + }) + }) + t.Run("an open degraded fault escalates", func(t *testing.T) { + mirror(t, usl, cls+"test_a_fault_already_open_degraded_escalates_when_the_hold_is_named", func(h *hl) { + h.unknownSend(true, true) + h.clock.Advance(20) + sw := h.sweeper() + batch, err := sw.Sweep(h.ctx, "crw") + mustDo(t, err) + var seen []faults.Observation + for _, o := range batch.Observations { + if o.FaultClass == "delivery_stalled" { + seen = append(seen, o) + } + } + keys := []any{} + for _, o := range seen { + keys = append(keys, o.OccurrenceKey) + } + h.eq(keys) + ledger := &faults.Ledger{Store: h.store, Clock: h.clock} + mustDo(t, sw.RecordAll(h.ctx, ledger, batch)) + for _, n := range []int{1, 2} { + o := seen[0] + o.OccurrenceKey = "delivery:earlier-" + string(rune('0'+n)) + _, err := ledger.Record(h.ctx, o) + mustDo(t, err) + } + h.eq(h.faultStates()) + h.ticks(1, 120) + h.assertBrokenNaming(UnknownSendLost) + escalated := publishedWith(h.publications(), "escalate:broken") + h.eq(len(escalated) > 0) + h.eq(regexp.MustCompile("is held: " + UnknownSendLost).MatchString(escalated[len(escalated)-1])) + }) + }) + t.Run("named before any sweep", func(t *testing.T) { + mirror(t, usl, cls+"test_a_hold_named_before_any_sweep_keeps_its_name_through_later_sweeps", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.clock.Advance(700) + h.assertBrokenNaming(UnknownSendLost) + h.eq(h.keys()) + h.eq(h.stall().I("occurrence_count")) + }) + }) + t.Run("two host losses", func(t *testing.T) { + mirror(t, usl, cls+"test_a_host_lost_turn_held_after_two_losses_keeps_its_keys", func(h *hl) { + event, _, turn := h.dispatched() + h.hostLoses(turn, true) + h.ticks(1, 120) + second := h.attemptsFor(event)[1] + h.hostLoses(str(loadsObj(second.S("record")), "turnId"), true) + h.ticks(4, 120) + h.eq(h.row(event).Opt("hold_reason")) + h.faultStates() + h.eq(h.keys()) + }) + }) + t.Run("an unknown send after a host loss", func(t *testing.T) { + mirror(t, usl, cls+"test_an_unknown_send_held_after_a_host_loss_reads_both_attempts", func(h *hl) { + event, _, turn := h.dispatched() + h.hostLoses(turn, true) + h.host.script = []string{"transport_unknown"} + h.ticks(1, 120) + second := h.attemptsFor(event)[1].S("request_id") + h.host.threads[parent].status = "notLoaded" + h.ticks(3, 120) + h.assertHeld(event, second) + h.faultStates() + h.eq(h.keys()) + }) + }) +} + +func (h *hl) dropTurn(turn string) { + t := h.host.threads[parent] + kept := t.turns[:0:0] + for _, one := range t.turns { + if one.TurnID != turn { + kept = append(kept, one) + } + } + t.turns = kept + h.dropItems(turn) +} + +func Test21_USL21_a_hold_that_changes_name_stays_on_its_one_fault(t *testing.T) { + const cls = "AHoldReachesItsFaultWhateverTheSweepSawFirst." + t.Run("undecided then lost", func(t *testing.T) { + mirror(t, usl, cls+"test_a_hold_that_changes_name_updates_the_one_fault_it_is_on", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + h.publications() + h.host.startTurn(parent, "", "completed", "the operator asked something") + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.eq(h.keys()) + h.eq(h.count("SELECT COUNT(*) AS c FROM fault_ledger WHERE fault_class = 'delivery_stalled'")) + h.eq(h.publications()) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("returns to an earlier name", func(t *testing.T) { + mirror(t, usl, cls+"test_a_hold_that_returns_to_an_earlier_name_names_it_again", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + turn := h.host.startTurn(parent, "", "completed", "the operator asked something") + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.dropTurn(turn.TurnID) + h.ticks(1, 700) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + h.eq(h.keys()) + h.eq(h.stall().I("occurrence_count")) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("namings between two sweeps", func(t *testing.T) { + mirror(t, usl, cls+"test_namings_between_two_sweeps_leave_the_standing_name", func(h *hl) { + event, first := h.unknownSend(false, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + turn := h.host.startTurn(parent, "", "completed", "the operator asked something") + h.reconcile(first, h.host) + h.assertHeld(event, first) + h.dropTurn(turn.TurnID) + h.reconcile(first, h.host) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendUndecided) + h.eq(h.keys()) + h.eq(h.stall().I("occurrence_count")) + rows, err := all(h.ctx, h.store, "SELECT detail FROM journal WHERE kind = 'unknown_send_hold_named' AND subject = ? ORDER BY seq", first) + mustDo(t, err) + named := []any{} + for _, r := range rows { + named = append(named, field(loadsObj(r.S("detail")), "hold")) + } + h.eq(named) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("readings that keep the name", func(t *testing.T) { + mirror(t, usl, cls+"test_readings_that_keep_the_name_record_nothing_new", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 120) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.keys() + h.ticks(3, 700) + h.assertHeld(event, first) + h.assertBrokenNaming(UnknownSendLost) + h.eq(h.keys()) + h.eq(h.stall().I("occurrence_count")) + h.eq(h.sendsTo(parent)) + }) + }) +} + +func Test21_USL22_a_superseded_hold_names_the_supersession(t *testing.T) { + const cls = "ASupersededHoldNamesTheSupersession." + openGenerationTwo := func(h *hl) { + h.host.startTurn(child, "turn-dispatch-2", "inProgress", "") + h.openGeneration("dispatch-2", "needs_changes_revision", "turn-dispatch-2") + } + t.Run("held then a new generation", func(t *testing.T) { + mirror(t, usl, cls+"test_a_held_send_a_new_generation_replaced_names_the_supersession", func(h *hl) { + event, first := h.unknownSend(true, true) + h.ticks(1, 120) + h.assertHeld(event, first) + openGenerationTwo(h) + outcome := h.reconcile(first, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + _, has := get(outcome, "recovery") + h.eq(has) + h.eq(h.statusPair(event)) + h.ticks(3, 700) + h.eq(h.sendsTo(parent)) + }) + }) + t.Run("superseded before a hold is named", func(t *testing.T) { + mirror(t, usl, cls+"test_a_send_superseded_before_its_hold_is_named_reports_the_supersession", func(h *hl) { + event, first := h.unknownSend(true, true) + h.clock.Advance(20) + openGenerationTwo(h) + h.eq(h.statusPair(event)) + outcome := h.reconcile(first, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + _, has := get(outcome, "recovery") + h.eq(has) + h.eq(h.sendsTo(parent)) + }) + }) + answered := func(h *hl) (string, string) { + _, correction := h.correctionAfterNeedsChanges() + h.host.startTurn(child, "child-earlier", "completed", "") + h.clock.Advance(300) + h.host.script = []string{"transport_unknown"} + record := h.attemptOn(correction, h.host, nil) + h.host.threads[child].status = "notLoaded" + h.clock.Advance(120) + h.reconcile(str(record, "requestId"), h.host) + row := h.row(correction) + h.eq([]any{row.S("state"), row.Opt("hold_reason")}) + h.host.startTurn(child, "child-late", "failed", "") + _, err := BindAnchor(h.ctx, h.store, h.clock, h.rid, 2, "child-late") + mustDo(h.t, err) + payload := h.executionPayload(h.rid, 2, "failed", 1, turnRef{child, "child-late", "failed"}) + _, err = h.accept(payload, storeAcceptNone) + mustDo(h.t, err) + _, err = h.delivery.Enqueue(h.ctx, str(payload, "eventId"), "", "") + mustDo(h.t, err) + return correction, str(record, "requestId") + } + t.Run("an answered correction", func(t *testing.T) { + mirror(t, usl, cls+"test_a_held_correction_its_generation_answered_names_the_child_disposition", func(h *hl) { + _, request := answered(h) + outcome := h.reconcile(request, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + _, has := get(outcome, "recovery") + h.eq(has) + h.eq(h.sendsTo(child)) + }) + }) + t.Run("answered, then generation 3", func(t *testing.T) { + mirror(t, usl, cls+"test_an_answered_correction_a_later_generation_passed_keeps_its_answer", func(h *hl) { + correction, request := answered(h) + h.host.startTurn(child, "turn-dispatch-3", "inProgress", "") + h.openGeneration("dispatch-3", "needs_changes_revision", "turn-dispatch-3") + outcome := h.reconcile(request, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + _, has := get(outcome, "recovery") + h.eq(has) + h.eq(h.statusPair(correction)) + }) + }) +} + +// openGeneration is registry.open_generation (todo 25 ports the registry; the transaction body is +// delivery's OpenGenerationIn). +func (h *hl) openGeneration(dispatch, reason, turn string) { + h.t.Helper() + mustDo(h.t, h.store.Transaction(h.ctx, func(ctx context.Context, _ *sql.Conn) error { + _, err := OpenGenerationIn(ctx, h.store, h.clock, h.rid, dispatch, reason, turn) + return err + })) +} diff --git a/internal/relay/delivery/unknownsend_grant_test.go b/internal/relay/delivery/unknownsend_grant_test.go new file mode 100644 index 00000000..f4a6cf6d --- /dev/null +++ b/internal/relay/delivery/unknownsend_grant_test.go @@ -0,0 +1,145 @@ +package delivery + +import ( + "bufio" + "context" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// USL-23: an uncertain merge-turn grant is read as status reads it. The merge turn itself +// (request, promotion, grant, acknowledgement, regrant) is mergeturn.py's, todo 26's to port, so +// testdata/grantstage.py stages and answers it in Python on a store in the Go test's tree; the +// part this property is about - reconciling the held grant notice and reading its status - runs +// in Go over that same file (internal/relay/mergeturn is the todo-21 subset of the grant rule). +// Every value the Python test asserts is compared with the Go value at the same assertion. + +type grantStage struct { + t *testing.T + cmd *exec.Cmd + in io.WriteCloser + out *bufio.Scanner + staged struct { + Waiting, Turn, Event, Store string + Record map[string]any + Receipt map[string]any + Sends [][]any + Now float64 + Threads []string + } +} + +func startGrantStage(t *testing.T, tree string) *grantStage { + root := repoRoot(t) + script, _ := filepath.Abs("testdata/grantstage.py") + cmd := exec.Command("uv", "run", "--no-sync", "python", script, tree) + cmd.Dir = filepath.Join(root, "packages", "codex-session-relay") + home := t.TempDir() + cmd.Env = append(os.Environ(), "HOME="+home, "XDG_STATE_HOME="+filepath.Join(tree, "xdg-state"), "XDG_DATA_HOME="+filepath.Join(home, "data"), "XDG_CONFIG_HOME="+filepath.Join(home, "config"), "CODEX_HOME="+filepath.Join(home, "codex"), "TMPDIR="+home, + "PYTHONPATH="+filepath.Join(root, "packages", "codex-session-relay", "src")+":"+filepath.Join(root, "packages", "codex-session-relay")) + cmd.Stderr = os.Stderr + in, err := cmd.StdinPipe() + mustDo(t, err) + outPipe, err := cmd.StdoutPipe() + mustDo(t, err) + mustDo(t, cmd.Start()) + g := &grantStage{t: t, cmd: cmd, in: in, out: bufio.NewScanner(outPipe)} + g.out.Buffer(make([]byte, 1<<20), 1<<24) + t.Cleanup(func() { _ = in.Close(); _ = cmd.Wait() }) + if !g.out.Scan() { + t.Fatalf("grant stage printed nothing: %v", g.out.Err()) + } + mustDo(t, json.Unmarshal(g.out.Bytes(), &g.staged)) + return g +} + +func (g *grantStage) do(command string) { + g.t.Helper() + if _, err := io.WriteString(g.in, command+"\n"); err != nil { + g.t.Fatal(err) + } + if !g.out.Scan() { + g.t.Fatalf("grant stage stopped after %q: %v", command, g.out.Err()) + } +} + +// grantHost is a fake host that holds the staged transport receipt, so reconciliation reads +// what Python's adapter answered for the notice. +func grantHost(h *hl, g *grantStage) { + for _, thread := range g.staged.Threads { + if h.host.threads[thread] == nil { + h.host.addThread(thread) + } + } + request := g.staged.Record["requestId"].(string) + receipt := Obj{} + for _, k := range []string{"requestId", "operation", "status", "threadId", "retrySafe", "error"} { + if v, ok := g.staged.Receipt[k]; ok { + receipt = append(receipt, F{Key: k, Value: v}) + } + } + h.host.ledger[request] = receipt + for _, s := range g.staged.Sends { + h.host.sends = append(h.host.sends, fakeSend{s[0].(string), s[1].(string), s[2].(string), s[3].(string)}) + } +} + +func runGrant(t *testing.T, name string, answer func(h *hl, g *grantStage)) { + root := pythonCaptures(t, usl) + raw, err := os.ReadFile(filepath.Join(root, name, "capture.json")) + mustDo(t, err) + var python pyCapture + mustDo(t, json.Unmarshal(raw, &python)) + if len(python.Problems) > 0 { + t.Fatalf("the Python test itself failed: %s", python.Problems) + } + tree := t.TempDir() + g := startGrantStage(t, tree) + s, err := store.Open(context.Background(), g.staged.Store, "") + mustDo(t, err) + t.Cleanup(func() { _ = s.Close() }) + clock := &FakeClock{T: g.staged.Now} + f := &fixture{t: t, ctx: context.Background(), tree: tree, clock: clock, store: s, host: newFakeHost(clock)} + f.delivery = NewService(s, clock) + h := &hl{fixture: f, name: name, ack: NewAck(f.delivery), rc: NewReconciler(f.delivery), adapter: f.host, policy: defaultTick()} + grantHost(h, g) + event := g.staged.Event + request := g.staged.Record["requestId"].(string) + // uncertain_grant + h.eq(g.staged.Waiting) + h.eq(g.staged.Record["transportReceiptStatus"]) + h.clock.Advance(120) + g.do("advance 120") + h.reconcile(request, h.host) + row := h.row(event) + h.eq(row.S("state")) + h.eq(row.S("hold_reason") == UnknownSendLost || row.S("hold_reason") == UnknownSendUndecided) + answer(h, g) + outcome := h.reconcile(request, h.host) + h.eq([]any{field(outcome, "nextExpectedAction"), field(outcome, "reason")}) + _, has := get(outcome, "recovery") + h.eq(has) + h.eq(h.statusPair(event)) + if name == "AnUncertainGrantIsReadAsStatusReadsIt.test_an_uncertain_grant_answered_on_its_turn_reads_acknowledged" { + h.eq(h.row(event).S("state")) + } + h.eq(h.sendsTo(parent)) + _, _ = io.WriteString(g.in, "quit\n") + requireSameCaptures(t, h.got, python.Captures) +} + +func Test21_USL23_an_uncertain_grant_is_read_as_status_reads_it(t *testing.T) { + const cls = "AnUncertainGrantIsReadAsStatusReadsIt." + t.Run("answered on its turn", func(t *testing.T) { + runGrant(t, cls+"test_an_uncertain_grant_answered_on_its_turn_reads_acknowledged", func(h *hl, g *grantStage) { g.do("answer") }) + }) + t.Run("regranted meanwhile", func(t *testing.T) { + runGrant(t, cls+"test_an_uncertain_grant_regranted_meanwhile_reads_superseded", func(h *hl, g *grantStage) { g.do("regrant") }) + }) +} diff --git a/internal/relay/delivery/value.go b/internal/relay/delivery/value.go deleted file mode 100644 index 4ccfe647..00000000 --- a/internal/relay/delivery/value.go +++ /dev/null @@ -1,218 +0,0 @@ -package delivery - -import ( - "bytes" - "encoding/json" - "math" - "strconv" - "strings" -) - -// object is a decoded JSON object (a Python dict): key order kept, a repeated key keeping its -// first position and last value, as json.loads does. -type object []field - -type field struct { - key string - value any -} - -func (o object) get(key string) any { - for _, f := range o { - if f.key == key { - return f.value - } - } - return nil -} - -// decode is json.loads: nil for text that is not JSON. Integers stay json.Number, other -// numbers become float64, so str() can tell 1 from 1.0 as Python does. -func decode(text string) any { - decoder := json.NewDecoder(strings.NewReader(text)) - decoder.UseNumber() - value, err := decodeValue(decoder) - if err != nil { - return nil - } - if decoder.More() { - return nil - } - return value -} - -func decodeValue(decoder *json.Decoder) (any, error) { - token, err := decoder.Token() - if err != nil { - return nil, err - } - switch v := token.(type) { - case json.Delim: - if v == '[' { - items := []any{} - for decoder.More() { - item, err := decodeValue(decoder) - if err != nil { - return nil, err - } - items = append(items, item) - } - _, err := decoder.Token() - return items, err - } - out := object{} - for decoder.More() { - key, err := decoder.Token() - if err != nil { - return nil, err - } - item, err := decodeValue(decoder) - if err != nil { - return nil, err - } - name, _ := key.(string) - replaced := false - for i := range out { - if out[i].key == name { - out[i].value, replaced = item, true - } - } - if !replaced { - out = append(out, field{name, item}) - } - } - _, err := decoder.Token() - return out, err - case json.Number: - if strings.ContainsAny(string(v), ".eE") { - f, _ := strconv.ParseFloat(string(v), 64) - return f, nil - } - return v, nil - } - return token, nil -} - -// truthy is bool(value). -func truthy(value any) bool { - switch v := value.(type) { - case nil: - return false - case bool: - return v - case string: - return v != "" - case json.Number: - return strings.TrimLeft(string(v), "-0") != "" - case float64: - return v != 0 - case []any: - return len(v) > 0 - case object: - return len(v) > 0 - } - return true -} - -// pyStr is str(value) for a decoded JSON value, as an f-string renders it. -func pyStr(value any) string { - if s, ok := value.(string); ok { - return s - } - return pyRepr(value) -} - -// pyRepr is repr(value) for a decoded JSON value. -func pyRepr(value any) string { - switch v := value.(type) { - case nil: - return "None" - case bool: - if v { - return "True" - } - return "False" - case string: - return reprString(v) - case json.Number: - return string(v) + "" - case float64: - return reprFloat(v) - case []any: - parts := make([]string, len(v)) - for i, item := range v { - parts[i] = pyRepr(item) - } - return "[" + strings.Join(parts, ", ") + "]" - case object: - parts := make([]string, len(v)) - for i, f := range v { - parts[i] = reprString(f.key) + ": " + pyRepr(f.value) - } - return "{" + strings.Join(parts, ", ") + "}" - } - return "" -} - -// reprFloat is float.__repr__. -func reprFloat(f float64) string { - switch { - case math.IsNaN(f): - return "nan" - case math.IsInf(f, 1): - return "inf" - case math.IsInf(f, -1): - return "-inf" - } - magnitude := math.Abs(f) - if magnitude == 0 || magnitude >= 1e-4 && magnitude < 1e16 { - text := strconv.FormatFloat(f, 'f', -1, 64) - if !strings.Contains(text, ".") { - text += ".0" - } - return text - } - text := strconv.FormatFloat(f, 'e', -1, 64) - mantissa, exponent, _ := strings.Cut(text, "e") - sign, digits := exponent[:1], strings.TrimLeft(exponent[1:], "0") - if len(digits) < 2 { - digits = strings.Repeat("0", 2-len(digits)) + digits - } - return mantissa + "e" + sign + digits -} - -// reprString is repr() of a str: single quotes unless it holds one and no double quote. -func reprString(text string) string { - quote := byte('\'') - if strings.Contains(text, "'") && !strings.Contains(text, `"`) { - quote = '"' - } - var b bytes.Buffer - b.WriteByte(quote) - for _, r := range text { - switch { - case r == '\\': - b.WriteString(`\\`) - case r == rune(quote): - b.WriteByte('\\') - b.WriteByte(quote) - case r == '\n': - b.WriteString(`\n`) - case r == '\r': - b.WriteString(`\r`) - case r == '\t': - b.WriteString(`\t`) - case r < 0x20 || r == 0x7f: - b.WriteString(`\x` + hex2(int(r))) - default: - b.WriteRune(r) - } - } - b.WriteByte(quote) - return b.String() -} - -func hex2(n int) string { - const digits = "0123456789abcdef" - return string([]byte{digits[n>>4], digits[n&15]}) -} diff --git a/internal/relay/delivery/verify.go b/internal/relay/delivery/verify.go new file mode 100644 index 00000000..83e9613e --- /dev/null +++ b/internal/relay/delivery/verify.go @@ -0,0 +1,228 @@ +package delivery + +import ( + "slices" +) + +// Settings verification of a resume response (settings.TaskSettings.mismatches, +// approval_divergence, roots_narrowing): the detector the guarded send runs between +// thread/resume and turn/start. The wire half of that send - thread/read, the resume, the ledger +// and turn/start - is the bridge adapter's (todo 28), which calls VerifyResume. + +const ( + SettingsNotPreserved = "settings_not_preserved" + SettingUnobservable = "setting_unobservable" + EnvironmentsUnknown = "environments_unknown" + UnverifiablePermissionProfile = "unverifiable_permission_profile" + SettingsDifferAfterLoad = "settings_differ_after_load" + ApprovalDiffersFromRecord = "approval_policy_differs_from_record" + RuntimeRootsNarrower = "runtime_roots_narrower_than_record" +) + +func canonicalJSON(v any) string { return dumpsSorted(v) } + +func textListOf(v any) []string { + a, _ := v.([]any) + out := make([]string, 0, len(a)) + for _, x := range a { + s, _ := x.(string) + out = append(out, s) + } + return out +} + +func rootsWithin(returned, recorded any) bool { + allowed := textListOf(recorded) + for _, r := range textListOf(returned) { + if !slices.Contains(allowed, r) { + return false + } + } + return true +} + +func without(o Obj, key string) Obj { + var out Obj + for _, f := range o { + if f.Key != key { + out = append(out, f) + } + } + return out +} + +func sandboxWithin(returned, recorded Obj) bool { + if str(returned, "type") != "workspaceWrite" || str(recorded, "type") != "workspaceWrite" { + return false + } + if canonicalJSON(without(returned, "writableRoots")) != canonicalJSON(without(recorded, "writableRoots")) { + return false + } + got, _ := get(returned, "writableRoots") + allowed, _ := get(recorded, "writableRoots") + return textList(got) && textList(allowed) && rootsWithin(got, allowed) +} + +func normaliseEnvironments(v any) []any { + list, _ := v.([]any) + out := make([]any, 0, len(list)) + for _, e := range list { + o := append(Obj(nil), e.(Obj)...) + if roots, _ := get(o, "runtimeWorkspaceRoots"); roots == nil { + cwd, _ := get(o, "cwd") + o = set(o, "runtimeWorkspaceRoots", []any{cwd}) + } + out = append(out, o) + } + return out +} + +func environmentsWithin(returned, recorded []any) bool { + if len(returned) != len(recorded) { + return false + } + for i := range returned { + got, allowed := returned[i].(Obj), recorded[i].(Obj) + if canonicalJSON(without(got, "runtimeWorkspaceRoots")) != canonicalJSON(without(allowed, "runtimeWorkspaceRoots")) { + return false + } + g, _ := get(got, "runtimeWorkspaceRoots") + a, _ := get(allowed, "runtimeWorkspaceRoots") + if !rootsWithin(g, a) { + return false + } + } + return true +} + +func settingsFinding(code, field string, expected, returned any, extra ...F) Obj { + o := Obj{{Key: "code", Value: code}, {Key: "field", Value: field}, {Key: "expected", Value: expected}, {Key: "returned", Value: returned}} + return append(o, extra...) +} + +// Mismatches is TaskSettings.mismatches: ordered findings against a resume response. +func (t TaskSettings) Mismatches(response any, transmitted, loadedBefore bool) []any { + r, ok := response.(Obj) + if !ok { + return []any{settingsFinding(SettingUnobservable, "response", "a resume response object", nil, F{Key: "returnedShape", Value: pyTypeName(response)})} + } + recorded, _ := get(t.Data, "approvalPolicy") + returned, _ := get(r, "approvalPolicy") + if returned == nil { + return []any{settingsFinding(SettingUnobservable, "approvalPolicy", recorded, nil)} + } + if p, isText := returned.(string); !isText || !slices.Contains(CarriedApprovalPolicies, p) { + label := any("granular") + if isText { + label = p + } + return []any{settingsFinding(UnsupportedApprovalPolicy, "approvalPolicy", []any{"never", "on-request"}, label, F{Key: "returnedShape", Value: pyTypeName(returned)})} + } + var found []any + environments, _ := get(t.Data, "environments") + thread, _ := get(r, "thread") + if thread == nil { + thread = Obj{} + } + th, isObj := thread.(Obj) + if !isObj { + return append(found, settingsFinding(SettingUnobservable, "environments", environments, nil, F{Key: "returnedShape", Value: "thread is " + pyTypeName(thread)})) + } + returnedEnv, _ := get(th, "environments") + if returnedEnv == nil { + return append(found, settingsFinding(EnvironmentsUnknown, "environments", environments, nil)) + } + if where, what, bad := environmentsProblem(returnedEnv); bad { + return append(found, settingsFinding(SettingUnobservable, "environments", environments, nil, F{Key: "returnedShape", Value: "environments" + where + " " + what})) + } + got := normaliseEnvironments(returnedEnv) + narrowable := !transmitted || loadedBefore + expectedEnv := normaliseEnvironments(environments) + if !(narrowable && environmentsWithin(got, expectedEnv) || !narrowable && canonicalJSON(got) == canonicalJSON(expectedEnv)) { + found = append(found, settingsFinding(SettingsNotPreserved, "environments", expectedEnv, got)) + } + sandbox, _ := get(t.Data, "sandbox") + expectations := map[string]any{"sandbox": normalisePolicy(sandbox)} + for _, k := range []string{"cwd", "runtimeWorkspaceRoots", "model", "reasoningEffort"} { + expectations[k], _ = get(t.Data, k) + } + for _, field := range []string{"sandbox", "cwd", "runtimeWorkspaceRoots", "model", "reasoningEffort"} { + expected := expectations[field] + raw, _ := get(r, field) + if raw == nil { + found = append(found, settingsFinding(SettingUnobservable, field, expected, nil)) + continue + } + value := raw + if field == "sandbox" { + normalised := normalisePolicy(raw) + if normalised == nil || expected.(Obj) == nil { + found = append(found, settingsFinding(SettingsNotPreserved, field, expected, raw)) + continue + } + if narrowable && sandboxWithin(normalised, expected.(Obj)) { + continue + } + value = normalised + } + if field == "runtimeWorkspaceRoots" { + if !textList(raw) { + found = append(found, settingsFinding(SettingUnobservable, field, expected, nil, F{Key: "returnedShape", Value: shapeOf(raw)})) + continue + } + if narrowable && textList(expected) && rootsWithin(raw, expected) { + continue + } + } + if canonicalJSON(expected) != canonicalJSON(value) { + found = append(found, settingsFinding(SettingsNotPreserved, field, expected, value)) + } + } + profile, _ := get(r, "activePermissionProfile") + expectedProfile, _ := get(t.Data, "expectedPermissionProfile") + if profile == nil && expectedProfile != nil { + found = append(found, settingsFinding(SettingUnobservable, "activePermissionProfile", expectedProfile, nil)) + } else if profile != nil && canonicalJSON(profile) != canonicalJSON(expectedProfile) { + found = append(found, settingsFinding(UnverifiablePermissionProfile, "activePermissionProfile", expectedProfile, profile)) + } + return found +} + +// ApprovalDivergence is approval_divergence: a carried policy other than the recorded one. +func (t TaskSettings) ApprovalDivergence(response any) any { + r, ok := response.(Obj) + if !ok { + return nil + } + observed, _ := get(r, "approvalPolicy") + recorded, _ := get(t.Data, "approvalPolicy") + p, isText := observed.(string) + if observed == recorded || !isText || !slices.Contains(CarriedApprovalPolicies, p) { + return nil + } + return Obj{{Key: "code", Value: ApprovalDiffersFromRecord}, {Key: "field", Value: "approvalPolicy"}, {Key: "recorded", Value: recorded}, {Key: "observed", Value: observed}} +} + +// VerifyResume is the guarded send's step between thread/resume and turn/start: the refusal the +// receipt carries (rpcError, settingsFindings) or the notes an accepted send carries. +func VerifyResume(t TaskSettings, resumed any, statusBefore string) (rpcError Obj, findings []any, notes []any) { + transmitted := !t.SettingsFreeResume + findings = t.Mismatches(resumed, transmitted, transmitted && statusBefore == "idle") + if len(findings) > 0 { + first := findings[0].(Obj) + code := str(first, "code") + expected, _ := get(first, "expected") + returned, _ := get(first, "returned") + if !transmitted { + if code == SettingsNotPreserved { + code = SettingsDifferAfterLoad + } + return Obj{{Key: "code", Value: code}, {Key: "message", Value: code + ": " + str(first, "field") + " is " + pyReprValue(returned) + " on the loaded thread and " + pyReprValue(expected) + " in the record; nothing was transmitted and no turn was started"}}, findings, nil + } + return Obj{{Key: "code", Value: code}, {Key: "message", Value: code + ": " + str(first, "field") + " returned " + pyReprValue(returned) + "; message withheld"}}, findings, nil + } + if note := t.ApprovalDivergence(resumed); note != nil { + notes = append(notes, note) + } + return nil, nil, notes +} diff --git a/internal/relay/faults/json.go b/internal/relay/faults/json.go new file mode 100644 index 00000000..a5c23101 --- /dev/null +++ b/internal/relay/faults/json.go @@ -0,0 +1,176 @@ +// Package faults is the relay's operational fault ledger (faults.py, faultsweep.py). +// +// Subset ported for todo 21; todo 22 owns this package. Only what the delivery hold paths +// exercise is here: FaultLedger.record for the observations the sweep derives (identity, +// occurrences, the timeline, suppression, the state transition, the opening and escalation +// publications and the blocking notification), and the two sweep sources that read deliveries +// (delivery_faults, retry_faults) with the clears recovered() derives for their class. Every +// other source, adoption, workspaces, rescoping, reopening and the publication worker are todo +// 22's; a path that would need one of them is refused with an error naming it rather than +// answered differently from Python. +package faults + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strconv" + "strings" + "unicode/utf16" +) + +func sha256Hex(text string) string { + sum := sha256.Sum256([]byte(text)) + return hex.EncodeToString(sum[:]) +} + +// dumps is Python json.dumps(value, ensure_ascii=False, sort_keys=True); compact selects +// separators=(",", ":"). +func dumps(value any, compact bool) string { + var b strings.Builder + write(&b, value, compact) + return b.String() +} + +func write(b *strings.Builder, value any, compact bool) { + item, key := ", ", ": " + if compact { + item, key = ",", ":" + } + switch v := value.(type) { + case nil: + b.WriteString("null") + case bool: + b.WriteString(strconv.FormatBool(v)) + case string: + writeString(b, v) + case int: + b.WriteString(strconv.Itoa(v)) + case int64: + b.WriteString(strconv.FormatInt(v, 10)) + case float64: + b.WriteString(pyFloat(v)) + case json.Number: + b.WriteString(v.String()) + case map[string]any: + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + b.WriteByte('{') + for i, k := range keys { + if i > 0 { + b.WriteString(item) + } + writeString(b, k) + b.WriteString(key) + write(b, v[k], compact) + } + b.WriteByte('}') + case []any: + b.WriteByte('[') + for i, x := range v { + if i > 0 { + b.WriteString(item) + } + write(b, x, compact) + } + b.WriteByte(']') + case []string: + list := make([]any, len(v)) + for i, s := range v { + list[i] = s + } + write(b, list, compact) + default: + panic(fmt.Sprintf("faults: unsupported JSON value %T", value)) + } +} + +func writeString(b *strings.Builder, s string) { + b.WriteByte('"') + for _, r := range s { + switch { + case r == '"' || r == '\\': + b.WriteByte('\\') + b.WriteRune(r) + case r == '\n': + b.WriteString(`\n`) + case r == '\r': + b.WriteString(`\r`) + case r == '\t': + b.WriteString(`\t`) + case r == '\b': + b.WriteString(`\b`) + case r == '\f': + b.WriteString(`\f`) + case r < 0x20: + for _, u := range utf16.Encode([]rune{r}) { + fmt.Fprintf(b, "\\u%04x", u) + } + default: + b.WriteRune(r) + } + } + b.WriteByte('"') +} + +// pyFloat is float.__repr__. +func pyFloat(f float64) string { + switch { + case math.IsNaN(f): + return "NaN" + case math.IsInf(f, 1): + return "Infinity" + case math.IsInf(f, -1): + return "-Infinity" + } + m := math.Abs(f) + if m == 0 || m >= 1e-4 && m < 1e16 { + text := strconv.FormatFloat(f, 'f', -1, 64) + if !strings.Contains(text, ".") { + text += ".0" + } + return text + } + text := strconv.FormatFloat(f, 'e', -1, 64) + mantissa, exponent, _ := strings.Cut(text, "e") + sign := exponent[0] + digits := strings.TrimLeft(exponent[1:], "0") + if len(digits) < 2 { + digits = fmt.Sprintf("%02s", digits) + } + return mantissa + "e" + string(sign) + digits +} + +// loads decodes JSON into map[string]any / []any / json.Number / string / bool / nil. +func loads(text string) (any, error) { + dec := json.NewDecoder(strings.NewReader(text)) + dec.UseNumber() + var v any + if err := dec.Decode(&v); err != nil { + return nil, err + } + return v, nil +} + +func loadsMap(text string) map[string]any { + v, err := loads(text) + if err != nil { + return map[string]any{} + } + m, _ := v.(map[string]any) + if m == nil { + return map[string]any{} + } + return m +} + +func named(v any) bool { + s, ok := v.(string) + return ok && strings.TrimSpace(s) != "" +} diff --git a/internal/relay/faults/ledger.go b/internal/relay/faults/ledger.go new file mode 100644 index 00000000..1754839e --- /dev/null +++ b/internal/relay/faults/ledger.go @@ -0,0 +1,585 @@ +package faults + +import ( + "context" + "database/sql" + "errors" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Words and bounds (faults.py). Subset ported for todo 21; todo 22 owns this. +const ( + SchemaObservation = "fault-observation/1" + idWidth = 32 + Notice = "notice" + Degraded = "degraded" + Broken = "broken" + Observed = "observed" + Open = "open" + FixPending = "fix_pending" + Resolved = "resolved" + Withdrawn = "withdrawn" + pending = "pending" + cancelled = "cancelled" + openRecord = "open_record" + appendComment = "append_comment" + blocking = "blocking" + triggerOpen = "open" + triggerEscalate = "escalate" + triggerRecur = "recur" + triggerReopen = "reopen" + occurrence = "occurrence" + clearedKind = "cleared" + maxEvidence = 8 + maxEvidenceBytes = 4096 + renderedOccur = 3 + defaultWindow = 21600.0 +) + +var severityRank = map[string]int{Notice: 0, Degraded: 1, Broken: 2} + +var productName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`) + +type classPolicy struct{ component, clears string } + +// classes are the registered classes this subset can record (register_class). +var classes = map[string]classPolicy{ + "delivery_stalled": {"delivery", "a delivery to the same recipient reaching dispatched"}, +} + +// ErrNotPorted marks a ledger path todo 22 ports; this subset refuses it rather than guessing. +var ErrNotPorted = errors.New("faults: not ported in the todo 21 subset (todo 22 owns this)") + +// Observation is faults.observation. +type Observation struct { + Product, FaultClass, Severity string + Signature map[string]any + OccurrenceKey string + Scope map[string]any + Detail string + Evidence []any + Cleared bool +} + +func canonicalSignature(signature map[string]any) string { return dumps(signature, true) } + +// FaultID is fault_id without a workspace. +func FaultID(product, class string, signature map[string]any) string { + return sha256Hex(product + "|" + class + "|" + canonicalSignature(signature))[:idWidth] +} + +func occurrenceID(identifier, key string, episode int64, cleared bool) string { + direction := "active" + if cleared { + direction = "cleared" + } + return sha256Hex(fmt.Sprintf("%s|%d|%s|%s", identifier, episode, direction, key))[:idWidth] +} + +func publicationID(identifier, kind, trigger string) string { + return sha256Hex(identifier + "|" + kind + "|" + trigger)[:idWidth] +} + +func identityDigest(identifier, kind, trigger string, cycle int64) string { + return sha256Hex(fmt.Sprintf("%s|%s|%s|%d", identifier, kind, trigger, cycle)) +} + +func evidenceDigest(evidence []any) string { return sha256Hex(dumps(evidence, true)) } + +// scopeKeyFor is scope_key_for without a workspace. +func scopeKeyFor(product string, scope map[string]any) string { + if p := scope["projectKey"]; named(p) { + return product + ":" + p.(string) + } + return product +} + +func boundedEvidence(evidence []any) ([]any, bool) { + kept := evidence + truncated := false + if len(kept) > maxEvidence { + kept, truncated = kept[:maxEvidence], true + } + kept = append([]any(nil), kept...) + for len(kept) > 0 && len(dumpsASCIIDefault(kept)) > maxEvidenceBytes { + kept = kept[:len(kept)-1] + truncated = true + } + return kept, truncated +} + +// dumpsASCIIDefault is json.dumps(kept, ensure_ascii=False) (insertion order, as Python measures +// it); the byte length of a sorted rendering is the same, which is all it is used for. +func dumpsASCIIDefault(v any) string { return dumps(v, false) } + +type fact struct { + Observation + id, component, signature, scopeKey, evidenceDigest string + evidence []any + truncated bool + threshold any + window float64 + publish bool +} + +func readObservation(o Observation) (fact, error) { + policy, ok := classes[o.FaultClass] + if !ok { + return fact{}, fmt.Errorf("%w: fault class %q", ErrNotPorted, o.FaultClass) + } + if _, ok := severityRank[o.Severity]; !ok || !productName.MatchString(o.Product) || !named(o.OccurrenceKey) || len(o.Signature) == 0 { + return fact{}, fmt.Errorf("faults: malformed observation") + } + if w, ok := o.Scope["workspace"]; ok && w != nil { + return fact{}, fmt.Errorf("%w: a workspace scope", ErrNotPorted) + } + scope := map[string]any{} + for k, v := range o.Scope { + if v != nil { + scope[k] = v + } + } + o.Scope = scope + evidence, truncated := boundedEvidence(o.Evidence) + threshold := map[string]any{Broken: int64(1), Degraded: int64(3), Notice: nil}[o.Severity] + return fact{Observation: o, id: FaultID(o.Product, o.FaultClass, o.Signature), component: policy.component, + signature: canonicalSignature(o.Signature), scopeKey: scopeKeyFor(o.Product, scope), evidence: evidence, + evidenceDigest: evidenceDigest(evidence), truncated: truncated, threshold: threshold, window: defaultWindow, publish: threshold != nil}, nil +} + +// Clock is the ledger's clock (Python's injected clock). +type Clock interface { + Now() float64 + ISO() string +} + +// Ledger is faults.FaultLedger. +type Ledger struct { + Store *store.Store + Clock Clock +} + +type row = store.Row + +func text(r row, name string) string { + switch v := r.Get(name).(type) { + case string: + return v + case []byte: + return string(v) + } + return "" +} + +func integer(r row, name string) int64 { + switch v := r.Get(name).(type) { + case int64: + return v + case float64: + return int64(v) + } + return 0 +} + +func (l *Ledger) one(ctx context.Context, query string, args ...any) (row, error) { + return l.Store.One(ctx, query, args...) +} + +func (l *Ledger) exec(ctx context.Context, query string, args ...any) (int64, error) { + result, err := l.Store.Q(ctx).ExecContext(ctx, query, args...) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// Record is FaultLedger.record without adoption: one observation converged on its fault, at +// most one write queued. The answer carries recorded and state; the rest is todo 22's surface. +func (l *Ledger) Record(ctx context.Context, o Observation) (bool, error) { + f, err := readObservation(o) + if err != nil { + return false, err + } + nowISO, now := l.Clock.ISO(), l.Clock.Now() + recorded := false + err = l.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + alias, err := l.one(ctx, "SELECT fault_id FROM fault_aliases WHERE alias_id = ?", f.id) + if err != nil { + return err + } + if alias != nil { + return fmt.Errorf("%w: an aliased fault", ErrNotPorted) + } + existing, err := l.one(ctx, "SELECT * FROM fault_ledger WHERE fault_id = ?", f.id) + if err != nil { + return err + } + if existing == nil && f.Cleared { + return nil + } + scopeText := dumps(f.Scope, false) + if existing == nil { + other, err := l.one(ctx, "SELECT fault_id, product FROM fault_ledger WHERE scope_key = ? AND product != ? LIMIT 1", f.scopeKey, f.Product) + if err == nil && other == nil { + other, err = l.one(ctx, "SELECT product FROM fault_target_projects WHERE scope_key = ? AND product != ?", f.scopeKey, f.Product) + } + if err != nil { + return err + } + if other != nil { + return fmt.Errorf("fault_scope_conflict: scope key %s is already carried by product %s", f.scopeKey, text(other, "product")) + } + if _, err := l.exec(ctx, "INSERT INTO fault_ledger (fault_id, product, fault_class, component, severity, signature, scope, scope_key, state, cycle, occurrence_count, reopen_count, detail, suppression, first_seen_at, last_seen_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,1,0,0,?,?,?,?,?)", + f.id, f.Product, f.FaultClass, f.component, f.Severity, f.signature, scopeText, f.scopeKey, Observed, f.Detail, nil, nowISO, nowISO, nowISO); err != nil { + return err + } + } else if f.scopeKey != text(existing, "scope_key") || scopeText != text(existing, "scope") { + return fmt.Errorf("%w: rescoping a fault", ErrNotPorted) + } + r, err := l.one(ctx, "SELECT * FROM fault_ledger WHERE fault_id = ?", f.id) + if err != nil { + return err + } + state := text(r, "state") + if f.Cleared && (state == Withdrawn || state == Resolved) { + return nil + } + episode := integer(r, "episode") + if !f.Cleared && r.Get("cleared_at") != nil { + episode++ + } + occurrenceKey := f.OccurrenceKey + oid := occurrenceID(f.id, occurrenceKey, episode, f.Cleared) + seen, err := l.one(ctx, "SELECT 1 FROM fault_timeline WHERE fault_id = ? AND ref_id = ?", f.id, oid) + if err != nil { + return err + } + storedKey := occurrenceKey + if f.Cleared { + storedKey += "#cleared" + } + if _, err := l.exec(ctx, "INSERT OR IGNORE INTO fault_occurrences (occurrence_id, fault_id, episode, occurrence_key, severity, cleared, detail, evidence, evidence_digest, truncated, observed_at, recorded_at, recorded_ts) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)", + oid, f.id, episode, storedKey, f.Severity, boolInt(f.Cleared), f.Detail, dumps(f.evidence, false), f.evidenceDigest, boolInt(f.truncated), nil, nowISO, now); err != nil { + return err + } + if seen != nil { + return nil + } + kind := occurrence + if f.Cleared { + kind = clearedKind + } + if _, err := l.exec(ctx, "INSERT INTO fault_timeline (fault_id, cycle, kind, ref_id, detail, recorded_at, recorded_ts) VALUES (?,?,?,?,?,?,?)", f.id, integer(r, "cycle"), kind, oid, f.Detail, nowISO, now); err != nil { + return err + } + count := integer(r, "occurrence_count") + if !f.Cleared { + count++ + } + severity := text(r, "severity") + if severityRank[f.Severity] > severityRank[severity] { + severity = f.Severity + } + escalated := severity != text(r, "severity") + f.threshold = map[string]any{Broken: int64(1), Degraded: int64(3), Notice: nil}[severity] + f.publish = f.threshold != nil + override, err := l.one(ctx, "SELECT threshold, window_seconds, reason, updated_at FROM fault_policies WHERE product = ? AND fault_class = ? AND severity = ?", f.Product, f.FaultClass, severity) + if err != nil { + return err + } + if override != nil { + return fmt.Errorf("%w: a policy override", ErrNotPorted) + } + suppression, publish, err := l.suppression(ctx, f.id, f, severity, now) + if err != nil { + return err + } + opened, create, err := issueSlot(ctx, l, r) + if err != nil { + return err + } + landed, err := l.one(ctx, "SELECT 1 FROM fault_publications WHERE fault_id = ? AND state IN (?,?,?) LIMIT 1", f.id, "issued", "uncertain", "confirmed") + if err != nil { + return err + } + next, cycle, reopened, trigger := transition(state, integer(r, "cycle"), f.Cleared, publish, escalated, severity, landed != nil, opened != "") + if next == Withdrawn || reopened || next == Resolved { + return fmt.Errorf("%w: a %s transition", ErrNotPorted, next) + } + detail := f.Detail + if detail == "" { + detail = text(r, "detail") + } + var clearedAt any + if f.Cleared { + clearedAt = nowISO + } + var resolvedAt any + if next == Resolved { + resolvedAt = r.Get("resolved_at") + } + if _, err := l.exec(ctx, "UPDATE fault_ledger SET state = ?, cycle = ?, severity = ?, episode = ?, occurrence_count = ?, reopen_count = reopen_count + ?, detail = ?, suppression = ?, last_seen_at = ?, cleared_at = ?, resolved_at = ?, updated_at = ? WHERE fault_id = ?", + next, cycle, severity, episode, count, 0, detail, suppression, nowISO, clearedAt, resolvedAt, nowISO, f.id); err != nil { + return err + } + if trigger != "" { + if err := l.enqueue(ctx, f.id, trigger, nowISO, opened, create, classes[f.FaultClass].clears); err != nil { + return err + } + reason, _, _ := strings.Cut(trigger, ":") + if (reason == triggerOpen || reason == triggerReopen) && severity == Broken { + if err := l.notify(ctx, f.id, cycle, nowISO); err != nil { + return err + } + } + } + recorded = true + return nil + }) + return recorded, err +} + +func boolInt(b bool) int64 { + if b { + return 1 + } + return 0 +} + +func (l *Ledger) suppression(ctx context.Context, identifier string, f fact, severity string, now float64) (string, bool, error) { + if !f.publish { + return dumps(map[string]any{"publish": false, "threshold": nil, "window": f.window, "counted": nil, + "reason": fmt.Sprintf("a %s is recorded for an operator and never filed", severity)}, false), false, nil + } + r, err := l.one(ctx, "SELECT COUNT(*) AS n FROM fault_timeline WHERE fault_id = ? AND kind = ? AND recorded_ts >= ?", identifier, occurrence, now-f.window) + if err != nil { + return "", false, err + } + counted := integer(r, "n") + threshold := f.threshold.(int64) + publish := counted >= threshold + reason := fmt.Sprintf("%d observation(s) inside %ds is under the threshold of %d", counted, int64(f.window), threshold) + if publish { + reason = fmt.Sprintf("%d observation(s) inside %ds reached the threshold of %d", counted, int64(f.window), threshold) + } + return dumps(map[string]any{"publish": publish, "threshold": threshold, "window": f.window, "counted": counted, "reason": reason}, false), publish, nil +} + +// issueSlot is _issue_slot: "issue", "create", or "" when free (then create may be a cancelled +// create row). +func issueSlot(ctx context.Context, l *Ledger, fault row) (string, row, error) { + if text(fault, "external_ref") != "" { + return "issue", nil, nil + } + create, err := l.one(ctx, "SELECT * FROM fault_publications WHERE fault_id = ? AND kind = ?", text(fault, "fault_id"), openRecord) + if err != nil { + return "", nil, err + } + if create != nil && text(create, "state") != cancelled { + return "create", create, nil + } + return "", create, nil +} + +// transition is _transition. +func transition(state string, cycle int64, cleared, publishable, escalated bool, severity string, landed, opened bool) (string, int64, bool, string) { + if cleared { + if landed || state == Resolved || state == Withdrawn { + return state, cycle, false, "" + } + return Withdrawn, cycle, false, "" + } + if state == Withdrawn { + state = Observed + } + switch state { + case Resolved: + if !opened { + if publishable { + return Open, cycle + 1, true, triggerOpen + } + return Open, cycle + 1, true, "" + } + return Open, cycle + 1, true, fmt.Sprintf("%s:%d", triggerReopen, cycle+1) + case FixPending: + if !opened { + if publishable { + return Open, cycle, false, triggerOpen + } + return Open, cycle, false, "" + } + return Open, cycle, false, fmt.Sprintf("%s:%d", triggerRecur, cycle) + case Open: + if !opened && publishable { + return Open, cycle, false, triggerOpen + } + if escalated { + return Open, cycle, false, triggerEscalate + ":" + severity + } + return Open, cycle, false, "" + } + if publishable { + return Open, cycle, false, triggerOpen + } + return Observed, cycle, false, "" +} + +// enqueue is _enqueue for a fault with no adoption and no issue: the one open_record, then +// comments (which need an issue this subset never has, so they are not queued). +func (l *Ledger) enqueue(ctx context.Context, identifier, trigger, now, holder string, create row, clears string) error { + adoption, err := l.one(ctx, "SELECT 1 FROM fault_adoptions WHERE fault_id = ? AND state = ?", identifier, pending) + if err != nil { + return err + } + reason, _, _ := strings.Cut(trigger, ":") + if reason == triggerOpen && adoption != nil { + return fmt.Errorf("%w: materializing an adoption", ErrNotPorted) + } + switch { + case reason == triggerOpen && holder == "issue", reason != triggerOpen && holder == "issue": + return fmt.Errorf("%w: a comment on an owned issue", ErrNotPorted) + case reason == triggerOpen && holder == "create": + return nil + case reason == triggerOpen: + return l.insertPublication(ctx, identifier, openRecord, triggerOpen, now, clears) + case holder == "": + return nil + } + // A live create holds the slot and the fault owns no issue yet: a comment is queued, and + // its target mode is none (KINDS[append_comment]). + return l.insertPublication(ctx, identifier, appendComment, trigger, now, clears) +} + +func (l *Ledger) insertPublication(ctx context.Context, identifier, kind, trigger, now, clears string) error { + fault, err := l.one(ctx, "SELECT * FROM fault_ledger WHERE fault_id = ?", identifier) + if err != nil { + return err + } + publication := publicationID(identifier, kind, trigger) + var team any + targetMode := "none" + if kind == openRecord { + targetMode = "team+project" + target, err := l.one(ctx, "SELECT t.tracker_ref, p.project_ref, p.product FROM fault_targets t LEFT JOIN fault_target_projects p ON p.scope_key = t.scope_key WHERE t.scope_key = ?", text(fault, "scope_key")) + if err != nil { + return err + } + if target != nil { + return fmt.Errorf("%w: a configured fault target", ErrNotPorted) + } + } + occurrences, err := l.Store.All(ctx, "SELECT * FROM fault_occurrences WHERE fault_id = ? ORDER BY rowid DESC LIMIT ?", identifier, renderedOccur) + if err != nil { + return err + } + summary := renderSummary(fault, trigger, occurrences, clears, publication) + existing, err := l.one(ctx, "SELECT state FROM fault_publications WHERE publication_id = ?", publication) + if err != nil { + return err + } + digest := identityDigest(identifier, kind, trigger, integer(fault, "cycle")) + switch { + case existing == nil: + if _, err := l.exec(ctx, "INSERT INTO fault_publications (publication_id, fault_id, kind, trigger_key, cycle, tracker_ref, external_ref, summary, identity_digest, state, attempts, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?,0,?,?)", + publication, identifier, kind, trigger, integer(fault, "cycle"), team, fault.Get("external_ref"), summary, digest, pending, now, now); err != nil { + return err + } + case text(existing, "state") == cancelled: + return fmt.Errorf("%w: reviving a cancelled publication", ErrNotPorted) + default: + return nil + } + _, err = l.exec(ctx, "INSERT INTO fault_publication_payloads (publication_id, project_ref, payload, hold_reason, updated_at, target_mode) VALUES (?,?,?,?,?,?) ON CONFLICT(publication_id) DO UPDATE SET project_ref = excluded.project_ref, payload = excluded.payload, hold_reason = excluded.hold_reason, updated_at = excluded.updated_at, target_mode = excluded.target_mode", + publication, nil, nil, nil, now, targetMode) + return err +} + +func (l *Ledger) notify(ctx context.Context, identifier string, cycle int64, now string) error { + fault, err := l.one(ctx, "SELECT product FROM fault_ledger WHERE fault_id = ?", identifier) + if err != nil { + return err + } + notification := sha256Hex(fmt.Sprintf("%s|%s|%d", identifier, blocking, cycle))[:idWidth] + _, err = l.exec(ctx, "INSERT INTO fault_notifications (notification_id, fault_id, product, kind, reason, cycle, ref, state, created_at, updated_at) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(notification_id) DO UPDATE SET state = excluded.state, last_error = NULL, updated_at = excluded.updated_at WHERE fault_notifications.state = ?", + notification, identifier, text(fault, "product"), blocking, nil, cycle, nil, pending, now, now, Withdrawn) + return err +} + +func domain(fault row) string { + signature, err := loads(text(fault, "signature")) + m, ok := signature.(map[string]any) + if err != nil || !ok { + return text(fault, "signature") + } + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, len(keys)) + for i, k := range keys { + parts[i] = k + "=" + pyStr(m[k]) + } + return strings.Join(parts, " ") +} + +func pyStr(v any) string { + switch t := v.(type) { + case nil: + return "None" + case string: + return t + case bool: + if t { + return "True" + } + return "False" + } + return fmt.Sprint(v) +} + +// renderSummary is render_summary for the open and escalate triggers this subset queues. +func renderSummary(fault row, trigger string, occurrences []row, clears, publication string) string { + lines := []string{fmt.Sprintf("[%s] %s: %s", text(fault, "product"), text(fault, "fault_class"), domain(fault)), ""} + reason, _, _ := strings.Cut(trigger, ":") + switch reason { + case triggerOpen: + lines = append(lines, fmt.Sprintf("The relay recorded a %s fault in its %s path and is filing it once.", text(fault, "severity"), text(fault, "component"))) + case triggerEscalate: + lines = append(lines, fmt.Sprintf("This fault escalated to %s.", text(fault, "severity"))) + } + lines = append(lines, "", "fault: "+text(fault, "fault_id"), + fmt.Sprintf("class: %s component: %s severity: %s", text(fault, "fault_class"), text(fault, "component"), text(fault, "severity")), + "domain: "+domain(fault), + fmt.Sprintf("observed: %d occurrence(s), first %s, most recent %s", integer(fault, "occurrence_count"), text(fault, "first_seen_at"), text(fault, "last_seen_at"))) + if clears != "" { + lines = append(lines, "clears when: "+clears) + } + if d := text(fault, "detail"); d != "" { + lines = append(lines, "detail: "+d) + } + if len(occurrences) > 0 { + lines = append(lines, "", "evidence, as observed at the time:") + for _, entry := range occurrences { + lines = append(lines, fmt.Sprintf("- %s %s", text(entry, "recorded_at"), text(entry, "occurrence_key"))) + evidence, _ := loads(text(entry, "evidence")) + items, _ := evidence.([]any) + for _, item := range items { + lines = append(lines, " "+dumps(item, false)) + } + if integer(entry, "truncated") != 0 { + lines = append(lines, " (evidence truncated at the recorded bound)") + } + } + } + lines = append(lines, "", "Recorded by codex-session-relay. The occurrence count counts observations, not incidents: a source that overwrites its own history is observed once per sweep, not once per failure.") + if publication != "" { + lines = append(lines, "publication: "+publication) + } + return strings.Join(lines, "\n") +} diff --git a/internal/relay/faults/sweep.go b/internal/relay/faults/sweep.go new file mode 100644 index 00000000..eae63354 --- /dev/null +++ b/internal/relay/faults/sweep.go @@ -0,0 +1,580 @@ +package faults + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// The delivery sources of faultsweep.sweep, and recorded()/record_all for them. Subset ported for +// todo 21; todo 22 owns this (the sync, observation, refusal, managed-start and reading sources, +// and every class's clear but delivery_stalled's, are its). + +const ( + sweepLimit = 32 + presentChecks = sweepLimit * 4 + busyCap = "busy_cap" + busyAttempt = "deferred_busy" + holdNamed = "unknown_send_hold_named" +) + +var ( + settledDelivery = []any{"dispatched", "inbox_only", "superseded"} + parentHolds = []string{"host_lost_turn", "unknown_send_lost", "unknown_send_undecided"} + unknownSendHolds = []string{"unknown_send_lost", "unknown_send_undecided"} + commit = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +const notSuperseded = "NOT EXISTS (SELECT 1 FROM delivery_supersession x WHERE x.event_id = d.event_id) AND NOT EXISTS (SELECT 1 FROM relationships sr WHERE sr.relationship_id = d.relationship_id AND sr.superseded_by IS NOT NULL)" + +// Installation is faultsweep.INSTALLATION: the package, its version and the directory of the +// installed copy, which every observation's facts carry. +type Installation struct { + Package, Version, Location string +} + +// Sweeper derives the delivery faults of one store. +type Sweeper struct { + Store *store.Store + // Current is delivery.supersession_reason(...) is None: whether a delivery still says + // something current (the send path's own rule, owned by the delivery package). + Current func(ctx context.Context, eventID string) (bool, error) + // MaxAttempts is the RetryPolicy's max_attempts. + MaxAttempts int64 + Installation Installation + // Now stamps the cursors, as faultsweep._now reads the wall clock. + Now func() string +} + +// Batch is one sweep's answer for the classes this subset derives. +type Batch struct { + Observations []Observation + Clears []Observation + positions map[string]any +} + +func pick(values ...any) []any { return values } + +// Sweep is faultsweep.sweep for delivery_stalled (its hold page and its retry page) and the +// clears recovered() derives for that class. +func (sw *Sweeper) Sweep(ctx context.Context, product string) (Batch, error) { + cursors, err := sw.readCursors(ctx) + if err != nil { + return Batch{}, err + } + held, err := sw.deliveryFaults(ctx, product, cursors["delivery_stalled"]) + if err != nil { + return Batch{}, err + } + retries, err := sw.retryFaults(ctx, product, cursors["delivery_retrying"]) + if err != nil { + return Batch{}, err + } + derived := append(append([]Observation(nil), held.observations...), retries.observations...) + clears, recoveredCursor, err := sw.recovered(ctx, product, cursors["recovered"]) + if err != nil { + return Batch{}, err + } + active := map[string]bool{} + for _, o := range derived { + active[o.FaultClass+"|"+canonicalSignature(o.Signature)] = true + } + var kept []Observation + for _, c := range clears { + if !active[c.FaultClass+"|"+canonicalSignature(c.Signature)] { + kept = append(kept, c) + } + } + positions := map[string]any{"delivery_stalled": held.cursor, "delivery_retrying": retries.cursor, "recovered": recoveredCursor} + return Batch{Observations: derived, Clears: kept, positions: positions}, nil +} + +// RecordAll is record_all: every observation recorded, then the cursors advanced. +func (sw *Sweeper) RecordAll(ctx context.Context, ledger *Ledger, batch Batch) error { + for _, o := range append(append([]Observation(nil), batch.Observations...), batch.Clears...) { + if _, err := ledger.Record(ctx, o); err != nil { + return err + } + } + return sw.writeCursors(ctx, batch.positions) +} + +func (sw *Sweeper) readCursors(ctx context.Context) (map[string]any, error) { + rows, err := sw.Store.All(ctx, "SELECT source, position FROM fault_cursors") + out := map[string]any{} + for _, r := range rows { + out[text(r, "source")] = r.Get("position") + } + return out, err +} + +func (sw *Sweeper) writeCursors(ctx context.Context, positions map[string]any) error { + now := sw.Now() + return sw.Store.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + for _, source := range []string{"delivery_stalled", "delivery_retrying", "recovered"} { + var stored any + if p := positions[source]; p != nil { + stored = dumps(p, false) + } + if _, err := sw.Store.Q(ctx).ExecContext(ctx, "INSERT INTO fault_cursors (source, position, pages, updated_at) VALUES (?,?,0,?) ON CONFLICT(source) DO UPDATE SET position = excluded.position, pages = 0, updated_at = excluded.updated_at", source, stored, now); err != nil { + return err + } + } + return nil + }) +} + +type page struct { + observations []Observation + cursor any + complete bool +} + +// rotation is _rotation for a text key, or an integer one. +func (sw *Sweeper) rotation(ctx context.Context, stored any, upper string, integerKey bool) (any, any, error) { + var at, until any + if s, ok := stored.(string); ok { + if v, err := loads(s); err == nil { + if m, ok := v.(map[string]any); ok { + at, until = m["at"], m["until"] + } else { + at = s + } + } else { + at = s + } + } + if until == nil { + r, err := sw.Store.One(ctx, upper) + if err != nil { + return nil, nil, err + } + if r != nil { + until = r[0].Value + } + } + if integerKey { + at, until = asInt(at), asInt(until) + } + if until == nil { + at = nil + } + return at, until, nil +} + +func asInt(v any) any { + switch n := v.(type) { + case json.Number: + i, _ := n.Int64() + return i + case float64: + return int64(n) + } + return v +} + +func pageOf(obs []Observation, rows []row, key string, after, until any) page { + filled := len(rows) >= sweepLimit + var cursor any + if filled { + cursor = map[string]any{"at": rows[len(rows)-1].Get(key), "until": until} + } + return page{obs, cursor, after == nil && !filled} +} + +func (sw *Sweeper) scopeOf(ctx context.Context, rid any, cache map[string]map[string]any) (map[string]any, error) { + id, _ := rid.(string) + if strings.TrimSpace(id) == "" { + return map[string]any{}, nil + } + if found, ok := cache[id]; ok { + return copyMap(found), nil + } + r, err := sw.Store.One(ctx, "SELECT r.issue_key, s.project_key FROM relationships r LEFT JOIN relationship_scope s ON s.relationship_id = r.relationship_id WHERE r.relationship_id = ?", id) + if err != nil { + return nil, err + } + found := map[string]any{} + if r != nil { + if p := text(r, "project_key"); p != "" { + found["projectKey"] = p + } + if k := text(r, "issue_key"); k != "" { + found["issueKey"] = k + } + } + cache[id] = found + return copyMap(found), nil +} + +func copyMap(m map[string]any) map[string]any { + out := map[string]any{} + for k, v := range m { + out[k] = v + } + return out +} + +func evidence(kind, ref string, observed map[string]any) map[string]any { + return map[string]any{"kind": kind, "ref": ref, "observed": observed} +} + +// facts is _facts, with installation() read for this copy. +func (sw *Sweeper) facts(expected, actual, impact string, limits []any, subject map[string]any) map[string]any { + installed := sw.installation() + stated := []any{"the installed revision is not known to the relay; the package version and the location of the installed copy identify it"} + if rev, ok := installed["revision"].(map[string]any); ok { + stated = nil + if rev["workingTreeClean"] == false { + stated = []any{"installed from a working tree with uncommitted changes, so the recorded commit does not fully identify the installed bytes"} + } + } + observed := map[string]any{"expected": expected, "actual": actual, "impact": impact, "installation": installed, "limits": append(limits, stated...)} + for k, v := range subject { + if v != nil { + observed[k] = v + } + } + return evidence("facts", "sweep", observed) +} + +// installation is faultsweep.installation: the revision the installer's host record +// attributes to this copy, or why it is unknown. +func (sw *Sweeper) installation() map[string]any { + base := os.Getenv("XDG_STATE_HOME") + root := filepath.Join(os.Getenv("HOME"), ".local", "state") + if base != "" { + root = base + } + path := filepath.Join(root, "codex-relay-workflow", "host-record.json") + answer := func(revision any, record any, reason any) map[string]any { + return map[string]any{"package": sw.Installation.Package, "version": sw.Installation.Version, "location": sw.Installation.Location, + "revision": revision, "revisionRecord": record, "revisionReason": reason} + } + unknown := func(reason string) map[string]any { return answer(nil, path, reason) } + raw, err := os.ReadFile(path) + switch { + case errors.Is(err, os.ErrNotExist): + return unknown("no host record at " + path) + case err != nil: + return unknown(fmt.Sprintf("the host record at %s could not be read: OSError", path)) + } + v, err := loads(string(raw)) + data, _ := v.(map[string]any) + if err != nil { + return unknown(fmt.Sprintf("the host record at %s is unreadable: JSONDecodeError", path)) + } + if n, ok := data["recordVersion"].(json.Number); !ok || n.String() != "1" { + return unknown("the host record is not record version 1") + } + components, _ := data["components"].(map[string]any) + relay, _ := components["codex-session-relay"].(map[string]any) + installs, ok := relay["installs"].([]any) + if !ok { + return unknown("the host record lists no codex-session-relay installs") + } + here, _ := filepath.EvalSymlinks(sw.Installation.Location) + var entry map[string]any + for _, raw := range installs { + e, _ := raw.(map[string]any) + location, _ := e["location"].(string) + if real, err := filepath.EvalSymlinks(location); e != nil && err == nil && real == here { + entry = e + } + } + if entry == nil { + return unknown("the host record has no install entry for " + sw.Installation.Location) + } + source, ok := entry["source"].(map[string]any) + if !ok { + return unknown("this copy's install entry records no revision (entries written before the installer recorded one per install carry none; the next install records it)") + } + var bad []string + for _, key := range []string{"repositoryCommit", "repositoryTree", "subdirectoryTree"} { + if s, ok := source[key].(string); !ok || !commit.MatchString(s) { + bad = append(bad, key) + } + } + if _, ok := source["workingTreeClean"].(bool); !ok { + bad = append(bad, "workingTreeClean") + } + if len(bad) > 0 { + return unknown("this copy's install entry records an incomplete revision (" + strings.Join(bad, ", ") + " missing or malformed), which identifies nothing") + } + revision := map[string]any{"environment": entry["environment"], "integrity": entry["integrity"]} + for _, key := range []string{"repositoryCommit", "repositoryTree", "subdirectoryTree", "workingTreeClean"} { + revision[key] = source[key] + } + return answer(revision, nil, nil) +} + +func (sw *Sweeper) current(ctx context.Context, event string, cache map[string]bool) (bool, error) { + if v, ok := cache[event]; ok { + return v, nil + } + v, err := sw.Current(ctx, event) + cache[event] = v + return v, err +} + +// settingsState refuses what _settings_items would read a settings hold for: the settings +// reading is the delivery/assignment port's, and the fault evidence for it todo 22's. +func settingsState(state string) error { + if state == "withheld_pre_send" || state == "inbox_only" { + return fmt.Errorf("%w: settings-hold evidence for a %s delivery", ErrNotPorted, state) + } + return nil +} + +func (sw *Sweeper) deliveryFaults(ctx context.Context, product string, cursor any) (page, error) { + after, until, err := sw.rotation(ctx, cursor, "SELECT MAX(event_id) FROM deliveries", false) + if err != nil || until == nil { + return page{complete: after == nil}, err + } + afterText, _ := after.(string) + rows, err := sw.Store.All(ctx, "SELECT d.event_id, d.relationship_id, d.recipient_task_id, d.state, d.hold_reason, d.attempt_count, e.execution_generation AS generation, e.turn_id AS turn, (SELECT a.request_id FROM attempts a WHERE a.event_id = d.event_id AND a.internal_state = 'settled' ORDER BY a.attempt_no DESC LIMIT 1) AS last_request, (SELECT a.state FROM attempts a WHERE a.event_id = d.event_id AND a.internal_state = 'settled' ORDER BY a.attempt_no DESC LIMIT 1) AS last_state FROM deliveries d LEFT JOIN events e ON e.event_id = d.event_id WHERE d.hold_reason IS NOT NULL AND d.hold_reason != ? AND d.state NOT IN (?,?,?) AND "+notSuperseded+" AND d.event_id > ? AND d.event_id <= ? ORDER BY d.event_id LIMIT ?", + append(append(pick(busyCap), settledDelivery...), afterText, until, sweepLimit)...) + if err != nil { + return page{}, err + } + var obs []Observation + scopes := map[string]map[string]any{} + cache := map[string]bool{} + for _, r := range rows { + ok, err := sw.current(ctx, text(r, "event_id"), cache) + if err != nil { + return page{}, err + } + if !ok { + continue + } + if err := settingsState(text(r, "state")); err != nil { + return page{}, err + } + hold := text(r, "hold_reason") + capped := integer(r, "attempt_count") >= sw.MaxAttempts + parked := contains(parentHolds, hold) + key := text(r, "last_request") + if key == "" { + key = text(r, "event_id") + } + occurrenceKey := "delivery:" + key + if contains(unknownSendHolds, hold) { + occurrenceKey += ":held:" + hold + if request := text(r, "last_request"); request != "" { + named, err := sw.Store.One(ctx, "SELECT MAX(seq) AS seq FROM journal WHERE kind = ? AND subject = ?", holdNamed, request) + if err != nil { + return page{}, err + } + if named != nil && named.Get("seq") != nil { + occurrenceKey += fmt.Sprintf(":%d", integer(named, "seq")) + } + } + } + severity := Degraded + if capped || parked { + severity = Broken + } + scope, err := sw.scopeOf(ctx, r.Get("relationship_id"), scopes) + if err != nil { + return page{}, err + } + recipient := text(r, "recipient_task_id") + obs = append(obs, Observation{Product: product, FaultClass: "delivery_stalled", Severity: severity, + Signature: map[string]any{"recipient": recipient, "attemptState": r.Get("last_state")}, OccurrenceKey: occurrenceKey, Scope: scope, + Detail: fmt.Sprintf("a delivery to %s is held: %s", recipient, hold), + Evidence: []any{evidence("row", "deliveries:"+text(r, "event_id"), map[string]any{"state": r.Get("state"), "holdReason": r.Get("hold_reason"), "attemptCount": r.Get("attempt_count"), "lastAttemptState": r.Get("last_state"), "relationship": r.Get("relationship_id")}), + sw.facts("the delivery reaches "+recipient, fmt.Sprintf("held (%s) after %d attempts; the last settled attempt ended %s", hold, integer(r, "attempt_count"), pyStr(r.Get("last_state"))), + "the recipient is not given this delivery while the hold stands", []any{"read from settled attempts only; one still in flight is not counted"}, + map[string]any{"event": r.Get("event_id"), "relationship": r.Get("relationship_id"), "generation": r.Get("generation"), "turn": r.Get("turn")})}}) + } + return pageOf(obs, rows, "event_id", after, until), nil +} + +func (sw *Sweeper) retryFaults(ctx context.Context, product string, cursor any) (page, error) { + after, until, err := sw.rotation(ctx, cursor, "SELECT MAX(rowid) FROM attempts", true) + if err != nil || until == nil { + return page{complete: after == nil}, err + } + afterInt, _ := after.(int64) + args := append(append([]any(nil), settledDelivery...), "dispatched", "inbox_only", busyAttempt, unknownSendHolds[0], unknownSendHolds[1], afterInt, until, sweepLimit) + rows, err := sw.Store.All(ctx, "SELECT a.rowid AS seq, a.request_id, a.state AS attempt_state, a.event_id, a.attempt_no, (SELECT MAX(x.attempt_no) FROM attempts x WHERE x.event_id = a.event_id AND x.internal_state = 'settled') AS latest_settled, d.relationship_id, d.recipient_task_id, d.state AS delivery_state, d.hold_reason, d.attempt_count, e.execution_generation AS generation, e.turn_id AS turn FROM attempts a JOIN deliveries d ON d.event_id = a.event_id LEFT JOIN events e ON e.event_id = a.event_id WHERE d.state NOT IN (?,?,?) AND a.internal_state = 'settled' AND a.state IS NOT NULL AND a.state NOT IN (?,?,?) AND NOT (COALESCE(d.hold_reason, '') IN (?,?) AND a.attempt_no = d.attempt_count) AND "+notSuperseded+" AND a.rowid > ? AND a.rowid <= ? ORDER BY a.rowid LIMIT ?", args...) + if err != nil { + return page{}, err + } + var obs []Observation + scopes := map[string]map[string]any{} + cache := map[string]bool{} + for _, r := range rows { + ok, err := sw.current(ctx, text(r, "event_id"), cache) + if err != nil { + return page{}, err + } + if !ok { + continue + } + cause, err := sw.attemptSettingsCause(ctx, text(r, "event_id"), text(r, "request_id")) + if err != nil { + return page{}, err + } + if cause != nil || text(r, "delivery_state") == "withheld_pre_send" { + return page{}, fmt.Errorf("%w: settings evidence on an attempt", ErrNotPorted) + } + hold := text(r, "hold_reason") + severity := Degraded + if hold != "" && hold != busyCap { + severity = Broken + } + scope, err := sw.scopeOf(ctx, r.Get("relationship_id"), scopes) + if err != nil { + return page{}, err + } + recipient, state := text(r, "recipient_task_id"), text(r, "attempt_state") + obs = append(obs, Observation{Product: product, FaultClass: "delivery_stalled", Severity: severity, + Signature: map[string]any{"recipient": recipient, "attemptState": state}, OccurrenceKey: "delivery:" + text(r, "request_id"), Scope: scope, + Detail: fmt.Sprintf("an attempt to deliver to %s ended %s", recipient, state), + Evidence: []any{evidence("row", "attempts:"+text(r, "request_id"), map[string]any{"attemptState": state, "deliveryState": r.Get("delivery_state"), "holdReason": r.Get("hold_reason"), "attemptCount": r.Get("attempt_count"), "event": r.Get("event_id")}), + sw.facts("the attempt reaches "+recipient, "the attempt ended "+state, "the delivery is retried and has not reached its recipient", []any{"one occurrence per settled failed attempt; attempts in flight are not read"}, + map[string]any{"event": r.Get("event_id"), "relationship": r.Get("relationship_id"), "generation": r.Get("generation"), "turn": r.Get("turn")})}}) + } + return pageOf(obs, rows, "seq", after, until), nil +} + +// attemptSettingsCause is _attempt_settings_cause. +func (sw *Sweeper) attemptSettingsCause(ctx context.Context, event, request string) (map[string]any, error) { + var best row + for _, q := range []struct { + sql string + args []any + }{ + {"SELECT seq, detail FROM journal WHERE subject = ? AND +kind = 'delivery_attempted' AND (CASE WHEN json_valid(detail) THEN json_extract(detail, '$.requestId') END) = ? ORDER BY seq DESC LIMIT 1", []any{event, request}}, + {"SELECT seq, detail FROM journal WHERE subject = ? AND +kind = 'reconciled' ORDER BY seq DESC LIMIT 1", []any{request}}, + } { + r, err := sw.Store.One(ctx, q.sql, q.args...) + if err != nil { + return nil, err + } + if r != nil && (best == nil || integer(r, "seq") > integer(best, "seq")) { + best = r + } + } + if best == nil { + return nil, nil + } + refusal, _ := loadsMap(text(best, "detail"))["settingsRefusal"].(map[string]any) + if _, ok := refusal["reason"].(string); ok { + return refusal, nil + } + return nil, nil +} + +func contains(list []string, v string) bool { + for _, x := range list { + if x == v { + return true + } + } + return false +} + +// recovered is recovered() for delivery_stalled: open faults of the class whose own source no +// longer produces them, asked of each fault directly (an exact existence query). +func (sw *Sweeper) recovered(ctx context.Context, product string, cursor any) ([]Observation, any, error) { + after, until, err := sw.rotation(ctx, cursor, "SELECT MAX(fault_id) FROM fault_ledger", false) + if err != nil || until == nil { + return nil, nil, err + } + afterText, _ := after.(string) + rows, err := sw.Store.All(ctx, "SELECT fault_id, fault_class, signature, cycle, scope FROM fault_ledger WHERE product = ? AND state IN (?,?,?) AND cleared_at IS NULL AND fault_class IN (?,?,?,?,?) AND fault_id > ? AND fault_id <= ? ORDER BY fault_id LIMIT ?", + product, Observed, Open, FixPending, "delivery_stalled", "record_sync_failed", "observation_stalled", "delivery_refused", "managed_start_failed", afterText, until, sweepLimit) + if err != nil { + return nil, nil, err + } + var clears []Observation + for _, r := range rows { + if text(r, "fault_class") != "delivery_stalled" { + continue + } + signature := loadsMap(text(r, "signature")) + present, err := sw.stillPresent(ctx, signature) + if err != nil { + return nil, nil, err + } + if present { + continue + } + last, err := sw.Store.One(ctx, "SELECT occurrence_id FROM fault_occurrences WHERE fault_id = ? AND cleared = 0 ORDER BY rowid DESC LIMIT 1", text(r, "fault_id")) + if err != nil { + return nil, nil, err + } + key := fmt.Sprintf("cleared:after:%d", integer(r, "cycle")) + if last != nil { + key = "cleared:after:" + text(last, "occurrence_id") + } + clears = append(clears, Observation{Product: product, FaultClass: "delivery_stalled", Severity: Notice, Signature: signature, OccurrenceKey: key, + Scope: loadsMap(text(r, "scope")), Cleared: true, Detail: "this sweep read the source and no longer derives this fault", + Evidence: []any{evidence("sweep", "delivery_stalled", map[string]any{"derived": false})}}) + } + var next any + if len(rows) >= sweepLimit { + next = map[string]any{"at": rows[len(rows)-1].Get("fault_id"), "until": until} + } + return clears, next, nil +} + +// stillPresent is still_present for delivery_stalled via _first_current. The overtaken-delivery +// memo (fault_overtaken_deliveries) is written as Python writes it. +func (sw *Sweeper) stillPresent(ctx context.Context, signature map[string]any) (bool, error) { + if signature["attemptState"] == busyAttempt { + return false, nil + } + state := signature["attemptState"] + query := "SELECT d.event_id FROM deliveries d WHERE d.recipient_task_id = ? AND d.state NOT IN (?,?,?) AND " + notSuperseded + " AND ((d.hold_reason IS NOT NULL AND d.hold_reason != ? AND COALESCE((SELECT a.state FROM attempts a WHERE a.event_id = d.event_id AND a.internal_state = 'settled' ORDER BY a.attempt_no DESC LIMIT 1), '') = COALESCE(?, '')) OR EXISTS (SELECT 1 FROM attempts a2 WHERE a2.event_id = d.event_id AND a2.internal_state = 'settled' AND a2.state = ?)) AND NOT EXISTS (SELECT 1 FROM fault_overtaken_deliveries o WHERE o.event_id = d.event_id) AND d.event_id > ? ORDER BY d.event_id LIMIT ?" + after, checked := "", 0 + var overtaken []string + for { + rows, err := sw.Store.All(ctx, query, append(append(pick(signature["recipient"]), settledDelivery...), busyCap, state, state, after, sweepLimit)...) + if err != nil { + return false, err + } + for _, r := range rows { + if checked >= presentChecks { + return false, fmt.Errorf("%w: a presence check past %d deliveries", ErrNotPorted, presentChecks) + } + checked++ + ok, err := sw.Current(ctx, text(r, "event_id")) + if err != nil { + return false, err + } + if ok { + return true, sw.noteOvertaken(ctx, overtaken) + } + overtaken = append(overtaken, text(r, "event_id")) + } + if len(rows) < sweepLimit { + return false, sw.noteOvertaken(ctx, overtaken) + } + after = text(rows[len(rows)-1], "event_id") + } +} + +func (sw *Sweeper) noteOvertaken(ctx context.Context, events []string) error { + if len(events) == 0 { + return nil + } + return fmt.Errorf("%w: noting overtaken deliveries", ErrNotPorted) +} + +// WallClockISO is faultsweep._now. +func WallClockISO() string { + return time.Now().UTC().Format("2006-01-02T15:04:05.000000+00:00") +} diff --git a/internal/relay/mergeturn/grant.go b/internal/relay/mergeturn/grant.go new file mode 100644 index 00000000..1ed41dc7 --- /dev/null +++ b/internal/relay/mergeturn/grant.go @@ -0,0 +1,163 @@ +// Package mergeturn is merge-turn coordination (mergeturn.py). +// +// Subset ported for todo 21; todo 26 owns this package. Only grant_supersession_in and the +// _current_grant_in it reads are here: what a merge-turn grant notice's own turn says about it, +// which delivery asks before a send, in status and in reconciliation. Requests, promotion, +// acknowledgement and every write are todo 26's. (internal/relay/cli/status.go carries todo 20's +// private copy of the same reading; todo 26 folds it onto this one.) +package mergeturn + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "strconv" + "strings" + + "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" +) + +// Why a grant notice has nothing left to tell (mergeturn.py). +const ( + Absent = "merge_turn_absent" + Closed = "merge_turn_closed" + Regranted = "merge_turn_regranted" + GrantAnswered = "merge_turn_grant_answered" + GrantUnreadable = "merge_turn_grant_unreadable" +) + +var occupying = []string{"holding", "merging", "unknown"} + +func text(r store.Row, name string) string { + switch v := r.Get(name).(type) { + case string: + return v + case []byte: + return string(v) + } + return "" +} + +func decode(raw string) map[string]any { + dec := json.NewDecoder(strings.NewReader(raw)) + dec.UseNumber() + var v any + if dec.Decode(&v) != nil { + return nil + } + m, _ := v.(map[string]any) + return m +} + +// GrantSupersessionFor is DeliveryService._grant_supersession: a grant notice's receipt names +// its turn and grant; "" means the notice is still current. +func GrantSupersessionFor(ctx context.Context, s *store.Store, receipt string) (string, error) { + envelope := decode(receipt) + if envelope == nil { + return GrantUnreadable, nil + } + turn, turnOK := envelope["turnId"].(string) + grant, grantOK := envelope["grantId"].(string) + if !turnOK || turn == "" || !grantOK || grant == "" { + return GrantUnreadable, nil + } + return GrantSupersessionIn(ctx, s, turn, grant) +} + +// GrantSupersessionIn is grant_supersession_in, read on ctx's querier. +func GrantSupersessionIn(ctx context.Context, s *store.Store, turn, grant string) (string, error) { + row, err := s.One(ctx, "SELECT state, tenure FROM merge_turns WHERE turn_id = ?", turn) + if err != nil { + return "", err + } + if row == nil { + return Absent, nil + } + answered, err := s.One(ctx, "SELECT 1 FROM merge_turn_ledger WHERE turn_id = ? AND idempotency_key = ?", turn, "grant_acknowledged:"+grant) + if err != nil { + return "", err + } + if answered != nil { + return GrantAnswered, nil + } + state := text(row, "state") + isOccupying := false + for _, o := range occupying { + isOccupying = isOccupying || o == state + } + if !isOccupying { + return Closed, nil + } + current, err := currentGrantIn(ctx, s, turn, row.Get("tenure")) + if err != nil { + return "", err + } + if current != "" && current != grant { + return Regranted, nil + } + return "", nil +} + +// currentGrantIn is MergeTurn._current_grant_in through grant_envelope: the newest well-formed +// grant of this tenure, "" when none is readable. +func currentGrantIn(ctx context.Context, s *store.Store, turn string, tenure any) (string, error) { + rows, err := s.All(ctx, "SELECT evidence, evidence_kind, idempotency_key FROM merge_turn_ledger WHERE turn_id = ? AND evidence_kind = ?", turn, "grant") + if err != nil { + return "", err + } + best, bestSequence := "", int64(math.MinInt64) + for _, row := range rows { + envelope := decode(text(row, "evidence")) + if envelope == nil { + continue + } + sequence, ok := integer(envelope["sequence"]) + if !ok || envelope["turnId"] != turn || !sameNumber(envelope["tenure"], tenure) { + continue + } + sum := sha256.Sum256([]byte(turn + "|" + fmt.Sprint(tenure) + "|" + strconv.FormatInt(sequence, 10))) + derived := "mtg-" + hex.EncodeToString(sum[:])[:32] + if envelope["grantId"] != derived || text(row, "idempotency_key") != "grant:"+derived { + continue + } + if r, ok := envelope["recipientTaskId"].(string); !ok || r == "" { + continue + } + if c, ok := envelope["candidateHead"].(string); !ok || c == "" { + continue + } + if sequence > bestSequence { + best, bestSequence = derived, sequence + } + } + return best, nil +} + +func integer(v any) (int64, bool) { + n, ok := v.(json.Number) + if !ok { + return 0, false + } + i, err := n.Int64() + return i, err == nil +} + +// sameNumber is Python == between a JSON number and the tenure column. +func sameNumber(a, b any) bool { + n, ok := a.(json.Number) + if !ok { + return a == nil && b == nil + } + switch t := b.(type) { + case int64: + i, err := n.Int64() + return err == nil && i == t + case float64: + f, err := n.Float64() + return err == nil && f == t + } + return false +} diff --git a/internal/relay/store/receipt_admission.go b/internal/relay/store/receipt_admission.go index 5e726c80..263d92c4 100644 --- a/internal/relay/store/receipt_admission.go +++ b/internal/relay/store/receipt_admission.go @@ -45,27 +45,6 @@ func (in ReceiptIntake) activeGeneration(ctx context.Context, id string, number return relationship, generation, nil } -// admitTurn is _check_turn_identity with the AnchorOrExplicit strategy and no continuation -// claim: the thread must be the registered child, and the turn the anchor or a turn already -// explicitly admitted against this anchor. -func (s *Store) admitTurn(ctx context.Context, relationship Relationship, generation Generation, turn TurnReference) (string, error) { - if turn.ThreadID != relationship.ChildTaskID { - return "", refuse(ReasonUnassignedTurn, "turnRef names thread %q, but the registered child of this relationship is %q", turn.ThreadID, relationship.ChildTaskID) - } - if generation.DispatchTurnID.Valid && turn.TurnID == generation.DispatchTurnID.String { - return "anchor", nil - } - var evidence string - err := s.q(ctx).QueryRowContext(ctx, `SELECT t.evidence FROM generation_turns t JOIN generations g ON g.relationship_id=t.relationship_id AND g.execution_generation=t.execution_generation WHERE t.relationship_id=? AND t.execution_generation=? AND t.turn_id=? AND g.dispatch_turn_id IS NOT NULL AND g.dispatch_turn_id <> '' AND t.evidence = (?||g.dispatch_turn_id)`, relationship.ID, generation.Number, turn.TurnID, boundExplicitPrefix).Scan(&evidence) - if err == nil { - return "explicit_admission", nil - } - if !errors.Is(err, sql.ErrNoRows) { - return "", fmt.Errorf("admitted turn: %w", err) - } - return "", refuse(ReasonUnassignedTurn, "turn %q is not admitted to generation %d (anchor %s): a turn other than the anchor needs an explicit continuation admission naming the generation, its anchor, an actor and a reason", turn.TurnID, generation.Number, strconv.Quote(generation.DispatchTurnID.String)) -} - // DaemonObservation synthesizes a receipt from an observed terminal turn. Only failure and // interruption: a daemon cannot know a completed turn produced something reviewable. func (in ReceiptIntake) DaemonObservation(ctx context.Context, relationshipID string, turn TurnReference) (StoredReceipt, error) { @@ -80,7 +59,7 @@ func (in ReceiptIntake) DaemonObservation(ctx context.Context, relationshipID st if err != nil { return StoredReceipt{}, err } - if _, err := in.Store.admitTurn(ctx, relationship, generation, turn); err != nil { + if err := in.checkTurnIdentity(ctx, relationship, generation, turn, nil); err != nil { return StoredReceipt{}, err } event, err := EventID(relationshipID, int(generation.Number), NoDeliverable, turn.Status, turn.TurnID, nil) @@ -103,7 +82,7 @@ func (in ReceiptIntake) DaemonObservation(ctx context.Context, relationshipID st {"eventId", str(event)}, }} claim := ReceiptClaim{EventID: event, RelationshipID: relationshipID, Generation: generation.Number, RevisionHash: NoDeliverable, Outcome: ObservationOutcome(turn.Status), Producer: ProducerDaemon, Turn: turn, manifest: null, document: document} - return in.storeEvent(ctx, claim, sql.NullString{}) + return in.storeEvent(ctx, claim, sql.NullString{}, nil) } // RecordObservation is record_observation: the daemon's (thread, turn, terminal status) key diff --git a/internal/relay/store/receipt_continuation.go b/internal/relay/store/receipt_continuation.go new file mode 100644 index 00000000..48deb11e --- /dev/null +++ b/internal/relay/store/receipt_continuation.go @@ -0,0 +1,168 @@ +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "strings" +) + +// ReasonRevisionLineageInvalid is errors.RefusalReason.REVISION_LINEAGE_INVALID. +const ReasonRevisionLineageInvalid = "revision_lineage_invalid" + +// continuationClaim is admission.ContinuationClaim: what a child states when it completes on a +// turn other than the anchor. +type continuationClaim struct { + anchor, actor, reason string +} + +// parseContinuation is ContinuationClaim.from_record; a malformed claim is a malformed receipt. +func parseContinuation(raw []byte) (*continuationClaim, error) { + if raw == nil { + return nil, nil + } + value, err := decodeOrdered(raw) + if err != nil || value.isNull() { + if err == nil { + return nil, nil + } + return nil, refuse(ReasonMalformedReceipt, "a continuation claim is an object") + } + if value.kind != jsonObject { + return nil, refuse(ReasonMalformedReceipt, "a continuation claim is an object") + } + var missing []string + for _, field := range []string{"anchorTurnId", "actor", "reason"} { + if _, ok := value.field(field); !ok { + missing = append(missing, field) + } + } + if len(missing) > 0 { + sort.Strings(missing) + quoted := make([]string, len(missing)) + for i, m := range missing { + quoted[i] = PyRepr(m) + } + return nil, refuse(ReasonMalformedReceipt, "a continuation claim needs [%s]", strings.Join(quoted, ", ")) + } + claim := &continuationClaim{} + for _, field := range []string{"anchorTurnId", "actor", "reason"} { + v, _ := value.field(field) + text, ok := v.text() + if !ok || strings.TrimSpace(text) == "" { + return nil, refuse(ReasonMalformedReceipt, "continuation.%s must be a non-empty string", field) + } + switch field { + case "anchorTurnId": + claim.anchor = text + case "actor": + claim.actor = text + default: + claim.reason = text + } + } + return claim, nil +} + +// checkTurnIdentity is _check_turn_identity with the AnchorOrExplicit strategy and no adapter: +// the thread must be the registered child, and the turn the anchor, a turn already explicitly +// admitted against this anchor, or one the claim admits now (recorded before the rest is +// checked, as Python's record_admission is). +func (in ReceiptIntake) checkTurnIdentity(ctx context.Context, relationship Relationship, generation Generation, turn TurnReference, claim *continuationClaim) error { + s := in.Store + if turn.ThreadID != relationship.ChildTaskID { + return refuse(ReasonUnassignedTurn, "turnRef names thread %s, but the registered child of this relationship is %s", PyRepr(turn.ThreadID), PyRepr(relationship.ChildTaskID)) + } + anchor := generation.DispatchTurnID + if anchor.Valid && turn.TurnID == anchor.String { + return nil + } + var evidence string + err := s.q(ctx).QueryRowContext(ctx, `SELECT t.evidence FROM generation_turns t JOIN generations g ON g.relationship_id=t.relationship_id AND g.execution_generation=t.execution_generation WHERE t.relationship_id=? AND t.execution_generation=? AND t.turn_id=? AND g.dispatch_turn_id IS NOT NULL AND g.dispatch_turn_id <> '' AND t.evidence = (?||g.dispatch_turn_id)`, relationship.ID, generation.Number, turn.TurnID, boundExplicitPrefix).Scan(&evidence) + if err == nil { + return nil + } + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("admitted turn: %w", err) + } + anchorRepr := "None" + if anchor.Valid { + anchorRepr = PyRepr(anchor.String) + } + unadmitted := func(detail string) error { + return refuse(ReasonUnassignedTurn, "turn %s is not admitted to generation %d (anchor %s): %s", PyRepr(turn.TurnID), generation.Number, anchorRepr, detail) + } + if claim == nil { + return unadmitted("a turn other than the anchor needs an explicit continuation admission naming the generation, its anchor, an actor and a reason") + } + if !anchor.Valid || claim.anchor != anchor.String { + return unadmitted(fmt.Sprintf("the continuation claims anchor %s, but generation %d is anchored to %s", PyRepr(claim.anchor), generation.Number, anchorRepr)) + } + detail := claim.actor + ": " + claim.reason + " | corroboration=not_corroborated" + return s.Transaction(ctx, func(ctx context.Context, conn *sql.Conn) error { + var bound sql.NullString + if err := conn.QueryRowContext(ctx, `SELECT dispatch_turn_id FROM generations WHERE relationship_id=? AND execution_generation=?`, relationship.ID, generation.Number).Scan(&bound); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return refuse(ReasonUnknownGeneration, "admission needs an existing generation") + } + return err + } + if !bound.Valid || strings.TrimSpace(bound.String) == "" { + return refuse(ReasonUnboundGeneration, "admission needs a bound generation") + } + _, err := conn.ExecContext(ctx, `INSERT INTO generation_turns (relationship_id, execution_generation, turn_id, evidence, actor, detail, admitted_at) VALUES (?,?,?,?,?,?,?) ON CONFLICT(relationship_id, execution_generation, turn_id) DO UPDATE SET evidence=excluded.evidence, actor=excluded.actor, detail=excluded.detail, admitted_at=excluded.admitted_at WHERE generation_turns.evidence <> excluded.evidence`, + relationship.ID, generation.Number, turn.TurnID, boundExplicitPrefix+bound.String, "child", detail, in.Now()) + return err + }) +} + +// recordLineage is currency.record_lineage: only a revision declaring itself is refused here; +// forks and unknown predecessors are recorded and surface when the head is read. +func recordLineage(ctx context.Context, conn *sql.Conn, relationshipID string, generation int64, eventID, revision string, supersedes *string, now string) error { + var declared any + declaredBy := "undeclared" + if supersedes != nil && *supersedes != "" { + text := strings.TrimSpace(*supersedes) + if text != "" { + declared = text + declaredBy = "child_declared" + } + if text == revision { + return refuse(ReasonRevisionLineageInvalid, "a revision cannot supersede itself") + } + } + _, err := conn.ExecContext(ctx, `INSERT OR IGNORE INTO revision_lineage (relationship_id, execution_generation, event_id, revision_hash, supersedes_hash, declared_by, recorded_at) VALUES (?,?,?,?,?,?,?)`, relationshipID, generation, eventID, revision, declared, declaredBy, now) + return err +} + +// PyRepr is Python's repr() of a str. +func PyRepr(text string) string { + quote := "'" + if strings.Contains(text, "'") && !strings.Contains(text, `"`) { + quote = `"` + } + var b strings.Builder + b.WriteString(quote) + for _, r := range text { + switch { + case r == '\\': + b.WriteString(`\\`) + case string(r) == quote: + b.WriteString(`\` + quote) + case r == '\n': + b.WriteString(`\n`) + case r == '\r': + b.WriteString(`\r`) + case r == '\t': + b.WriteString(`\t`) + case r < 0x20 || r == 0x7f: + fmt.Fprintf(&b, `\x%02x`, r) + default: + b.WriteRune(r) + } + } + b.WriteString(quote) + return b.String() +} diff --git a/internal/relay/store/receipt_intake.go b/internal/relay/store/receipt_intake.go index 071b7592..2821455a 100644 --- a/internal/relay/store/receipt_intake.go +++ b/internal/relay/store/receipt_intake.go @@ -37,9 +37,23 @@ type StoredReceipt struct { // AcceptChildReceipt is accept_child_receipt: every refusal is recorded for an operator. func (in ReceiptIntake) AcceptChildReceipt(ctx context.Context, payload []byte, observation TurnReference) (StoredReceipt, error) { - stored, err := in.accept(ctx, payload, observation) + return in.AcceptChildReceiptWith(ctx, payload, observation, AcceptOptions{}) +} + +// AcceptOptions carries what accept_child_receipt takes beside the payload: a continuation +// admission (JSON, nil for none) and the revision a re-emission declares it supersedes. +type AcceptOptions struct { + Continuation []byte + SupersedesRevision *string +} + +// AcceptChildReceiptWith is accept_child_receipt with its continuation and supersedes_revision. +func (in ReceiptIntake) AcceptChildReceiptWith(ctx context.Context, payload []byte, observation TurnReference, options AcceptOptions) (StoredReceipt, error) { + stored, err := in.accept(ctx, payload, observation, options) if reason := RefusalReason(err); reason != "" { - refusal := Refusal{At: in.Now(), Reason: reason, Detail: sql.NullString{String: err.Error(), Valid: true}, Payload: sql.NullString{String: string(payload), Valid: true}} + var refused *RefusedError + errors.As(err, &refused) + refusal := Refusal{At: in.Now(), Reason: reason, Detail: sql.NullString{String: refused.Detail, Valid: true}, Payload: sql.NullString{String: string(payload), Valid: true}} if document, decodeErr := decodeOrdered(payload); decodeErr == nil && document.kind == jsonObject { if rid, ok := document.field("relationshipId"); ok { refusal.RelationshipID = sql.NullString{String: pythonStr(rid), Valid: !rid.isNull()} @@ -58,7 +72,7 @@ func (in ReceiptIntake) AcceptChildReceipt(ctx context.Context, payload []byte, return stored, err } -func (in ReceiptIntake) accept(ctx context.Context, payload []byte, observation TurnReference) (StoredReceipt, error) { +func (in ReceiptIntake) accept(ctx context.Context, payload []byte, observation TurnReference, options AcceptOptions) (StoredReceipt, error) { claim, err := ParseReceipt(payload) if err != nil { return StoredReceipt{}, err @@ -79,7 +93,11 @@ func (in ReceiptIntake) accept(ctx context.Context, payload []byte, observation if claim.Turn != observation { return StoredReceipt{}, refuse(ReasonTurnRefMismatch, "turnRef must equal the observation that accompanied this receipt") } - if _, err := in.Store.admitTurn(ctx, relationship, generation, claim.Turn); err != nil { + continuation, err := parseContinuation(options.Continuation) + if err != nil { + return StoredReceipt{}, err + } + if err := in.checkTurnIdentity(ctx, relationship, generation, claim.Turn, continuation); err != nil { return StoredReceipt{}, err } allowed := compatibleTurnStatus(claim.Outcome, claim.Turn.Status) @@ -105,7 +123,7 @@ func (in ReceiptIntake) accept(ctx context.Context, payload []byte, observation if claim.EventID != expected { return StoredReceipt{}, refuse(ReasonEventIDMismatch, "event id should be %s for these fields", expected) } - return in.storeEvent(ctx, claim, binding) + return in.storeEvent(ctx, claim, binding, options.SupersedesRevision) } // checkDeliverable branches on OUTCOME before producer: a reviewable receipt must verify its @@ -178,7 +196,7 @@ func (in ReceiptIntake) requireMinimum(mode PathBinding) (PathBinding, error) { } // storeEvent is _store_event: re-observing one revision is one fact seen twice. -func (in ReceiptIntake) storeEvent(ctx context.Context, claim ReceiptClaim, binding sql.NullString) (StoredReceipt, error) { +func (in ReceiptIntake) storeEvent(ctx context.Context, claim ReceiptClaim, binding sql.NullString, supersedes *string) (StoredReceipt, error) { now := in.Now() record, err := pythonDumps(claim.document) if err != nil { @@ -214,8 +232,7 @@ func (in ReceiptIntake) storeEvent(ctx context.Context, claim ReceiptClaim, bind return nil } // Atomic with the receipt: a revision stored without its declaration reads as undeclared. - _, err := conn.ExecContext(ctx, `INSERT OR IGNORE INTO revision_lineage (relationship_id,execution_generation,event_id,revision_hash,supersedes_hash,declared_by,recorded_at) VALUES (?,?,?,?,NULL,'undeclared',?)`, claim.RelationshipID, claim.Generation, claim.EventID, claim.RevisionHash, now) - return err + return recordLineage(ctx, conn, claim.RelationshipID, claim.Generation, claim.EventID, claim.RevisionHash, supersedes, now) }) return result, err } diff --git a/internal/relay/store/records.go b/internal/relay/store/records.go index 5dc10bea..3db9a76a 100644 --- a/internal/relay/store/records.go +++ b/internal/relay/store/records.go @@ -272,3 +272,11 @@ func (s *Store) AppendJournal(ctx context.Context, entry JournalEntry) error { return err }) } + +// Querier is where a read or a write outside a transaction body runs. +type Querier = querier + +// Q exposes the ctx-aware querier to domain packages: the open transaction's connection when ctx +// carries one of this store's, otherwise the pool. Readers in other packages go through it so +// they see the transaction's own uncommitted writes, as Python's one connection does. +func (s *Store) Q(ctx context.Context) Querier { return s.q(ctx) } diff --git a/internal/relay/store/registration_hold.go b/internal/relay/store/registration_hold.go new file mode 100644 index 00000000..6dccb964 --- /dev/null +++ b/internal/relay/store/registration_hold.go @@ -0,0 +1,49 @@ +package store + +import ( + "context" + "database/sql" + "net/url" + "time" +) + +// RegistrationTimeout is intent.SQLITE_TIMEOUT: the lock wait a registration hold, or a read-only +// open of the relay store by the marker readers, may spend. Declared once, beside the opens. +const RegistrationTimeout = 2 * time.Second + +// RegistrationHold is intent.registration_hold: the relay's write lock (BEGIN IMMEDIATE) held +// across a check and the marker publication that depends on it, and never committed. It goes +// through the store's write admission (refuseLiveState, and the ownership fence todo 30 adds to +// the same path) before any connection is made, opens mode=rw and never rwc, so an absent store +// stays absent, and ends in ROLLBACK so it records nothing. +// +// run receives the held connection; why is "" when the hold was taken, else the refusal text +// Python yields beside a None connection, byte for byte. +func RegistrationHold(ctx context.Context, dbPath string, run func(conn *sql.Conn, why string) error) error { + absolute, err := expandUser(dbPath) + if err != nil { + return run(nil, "the relay store path "+pythonRepr(dbPath)+" could not be read as a path") + } + if absolute, err = refuseLiveState(absolute); err != nil { + return run(nil, "the relay store could not be opened for writing: "+err.Error()) + } + u := url.URL{Scheme: "file", Path: absolute} + db, err := boundedDB(u.Path, "rw", RegistrationTimeout) + if err != nil { + return run(nil, "the relay store could not be opened for writing: "+PythonSQLiteMessage(err)) + } + defer db.Close() + conn, err := db.Conn(ctx) + if err != nil { + return run(nil, "the relay store could not be opened for writing: "+PythonSQLiteMessage(err)) + } + defer conn.Close() + if _, err := conn.ExecContext(ctx, "BEGIN IMMEDIATE"); err != nil { + return run(nil, "the relay store's write lock could not be taken: "+PythonSQLiteMessage(err)) + } + runErr := run(conn, "") + // ROLLBACK and never COMMIT: the hold writes nothing of its own. A failed ROLLBACK is + // Python's `except sqlite3.Error: pass`: closing the connection ends the transaction anyway. + _, _ = conn.ExecContext(context.WithoutCancel(ctx), "ROLLBACK") + return runErr +} From 820c11076bf69741b41faa66f123e4b3f086442c Mon Sep 17 00:00:00 2001 From: thisisjun786 <259586770+thisisjun786@users.noreply.github.com> Date: Sun, 27 Sep 2026 00:04:29 +0900 Subject: [PATCH 2/3] fix(relay/delivery): race-free anchor binding, recover acknowledged anchors, validate --kind-module --- docs/port/known-defects.md | 4 ++ internal/relay/cli/python_parity_test.go | 11 +++++ internal/relay/cli/registry.go | 32 +++++++++---- internal/relay/delivery/ack.go | 49 +++++++++++--------- internal/relay/delivery/anchor_test.go | 58 ++++++++++++++++++++++++ internal/relay/delivery/cli.go | 10 ++++ 6 files changed, 134 insertions(+), 30 deletions(-) create mode 100644 docs/port/known-defects.md diff --git a/docs/port/known-defects.md b/docs/port/known-defects.md new file mode 100644 index 00000000..fe6142f7 --- /dev/null +++ b/docs/port/known-defects.md @@ -0,0 +1,4 @@ +# Python defects not carried over + +- **Python defect not carried over:** `registry.bind_anchor` (`registry.py:790-822`) reads the pending anchor before beginning its transaction. Concurrent binders can overwrite a generation's dispatch turn after one has already succeeded, violating the never-rebind invariant. Go reads and decides inside the write transaction; `TestBindAnchor_concurrent_turns_never_replace_the_winner` proves only one distinct turn binds and the loser is refused. +- **Python defect not carried over:** `ack.bind_dispatched_revision` (`ack.py:1026-1035`) rejects acknowledged revisions even though `ack.bind_pending_anchors` (`ack.py:1037-1060`) selects them for recovery. This violates the invariant that every dispatched revision's pending anchor can recover after acknowledgement. Go accepts both states; `TestBindPendingAnchors_recovers_acknowledged_revision` proves the recovery binds and is idempotent. diff --git a/internal/relay/cli/python_parity_test.go b/internal/relay/cli/python_parity_test.go index c959832a..4550f1ec 100644 --- a/internal/relay/cli/python_parity_test.go +++ b/internal/relay/cli/python_parity_test.go @@ -250,6 +250,17 @@ func TestKindModule_matches_python_for_an_unimportable_module(t *testing.T) { } } +func TestDelivery_kind_module_refusal_matches_python_before_ack_proof(t *testing.T) { + home := pythonHome(t) + args := []string{"--kind-module", "does_not_exist", "ack-proof", "--event", "0123456789abcdef0123456789abcdef", "--turn", "turn-1"} + for _, argv := range [][]string{args, args[2:]} { + py, got := python(t, home, argv...), golang(t, home, argv...) + if py.code != got.code || py.stdout != got.stdout || py.stderr != got.stderr { + t.Fatalf("%v: python %+v; go %+v", argv, py, got) + } + } +} + func lastLine(text string) string { lines := strings.Split(strings.TrimRight(text, "\n"), "\n") return lines[len(lines)-1] diff --git a/internal/relay/cli/registry.go b/internal/relay/cli/registry.go index 5d0208c4..cf7f114a 100644 --- a/internal/relay/cli/registry.go +++ b/internal/relay/cli/registry.go @@ -120,15 +120,17 @@ func ExecuteAs(ctx context.Context, argv0 string, argv []string, stdout, stderr } if slices.Contains(delivery.CommandNames(), remaining[0]) { code, _ := delivery.ExecuteAs(ctx, prog, argv, stdout, stderr, func(selection store.StateSelection, socket string) error { - services := Services{Selection: selection, SocketPath: socket, AdapterRequested: socket != "", Program: program(argv0)} - refusal, err := selectionRefusal(services) - if err != nil { - return err + if remaining[0] != "ack-proof" { + services := Services{Selection: selection, SocketPath: socket, AdapterRequested: socket != "", Program: program(argv0)} + refusal, err := selectionRefusal(services) + if err != nil { + return err + } + if refusal != nil { + return &PayloadExit{Payload: refusal, Code: contract.ExitRefused} + } } - if refusal != nil { - return &PayloadExit{Payload: refusal, Code: contract.ExitRefused} - } - return nil + return kindModuleRefusal(*kindModules) }) return code } @@ -219,6 +221,20 @@ func importKindModules(names []string) error { return nil } +// Delegated commands run their selection check before importing modules. Give their +// runners the same complete CLI error envelope as the ordinary command path. +func kindModuleRefusal(names []string) error { + err := importKindModules(names) + if err == nil { + return nil + } + var usage *UsageError + if errors.As(err, &usage) { + return &PayloadExit{Payload: contract.OrderedObject{{Key: "error", Value: "usage"}, {Key: "detail", Value: usage.Detail}}, Code: usage.Code} + } + return &PayloadExit{Payload: contract.OrderedObject{{Key: "error", Value: "host"}, {Key: "detail", Value: err.Error()}}, Code: contract.ExitHost} +} + func emit(stdout, stderr io.Writer, result any, err error) int { var usage *UsageError var payload *PayloadExit diff --git a/internal/relay/delivery/ack.go b/internal/relay/delivery/ack.go index 02599a0b..236ce324 100644 --- a/internal/relay/delivery/ack.go +++ b/internal/relay/delivery/ack.go @@ -776,7 +776,7 @@ func (a *Ack) settlePendingAck(ctx context.Context, eventID string, pending Row, // BindDispatchedRevision is bind_dispatched_revision. func (a *Ack) BindDispatchedRevision(ctx context.Context, revisionEvent string) (Obj, error) { row, err := a.Delivery.Find(ctx, revisionEvent) - if err != nil || row == nil || row.S("state") != Dispatched || !truthy(row.Opt("dispatch_turn_id")) { + if err != nil || row == nil || !slices.Contains([]string{Dispatched, Acknowledged}, row.S("state")) || !truthy(row.Opt("dispatch_turn_id")) { return nil, err } event, err := a.Delivery.eventRow(ctx, revisionEvent) @@ -791,32 +791,37 @@ func BindAnchor(ctx context.Context, s *store.Store, clock Clock, rid string, nu if strings.TrimSpace(turn) == "" { return nil, refuse("unbound_generation", "an anchor needs an exact dispatch turn id") } - r, err := LoadRelationship(ctx, s, rid) - if err != nil { - return nil, err - } - current := r.generation(number) - if current == nil { - return nil, refuse(UnknownGeneration, "%s has no generation %d", store.PyRepr(rid), number) - } - if current.S("anchor_state") == "bound" { - if current.S("dispatch_turn_id") == turn { - return generationRecord(current), nil + var bound Obj + err := s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + r, err := LoadRelationship(ctx, s, rid) + if err != nil { + return err } - return nil, refuse("anchor_already_bound", "generation %d is already bound to %s", number, pyReprValue(current.Opt("dispatch_turn_id"))) - } - now := clock.ISO() - err = s.Transaction(ctx, func(ctx context.Context, _ *sql.Conn) error { + current := r.generation(number) + if current == nil { + return refuse(UnknownGeneration, "%s has no generation %d", store.PyRepr(rid), number) + } + if current.S("anchor_state") == "bound" { + if current.S("dispatch_turn_id") == turn { + bound = generationRecord(current) + return nil + } + return refuse("anchor_already_bound", "generation %d is already bound to %s", number, pyReprValue(current.Opt("dispatch_turn_id"))) + } + now := clock.ISO() if _, err := execSQL(ctx, s, "UPDATE generations SET anchor_state = ?, dispatch_turn_id = ?, bound_at = ? WHERE relationship_id = ? AND execution_generation = ?", "bound", turn, now, rid, number); err != nil { return err } - return journal(ctx, s, "anchor_bound", rid, Obj{{Key: "generation", Value: number}}, now) + if err := journal(ctx, s, "anchor_bound", rid, Obj{{Key: "generation", Value: number}}, now); err != nil { + return err + } + current, err = one(ctx, s, "SELECT * FROM generations WHERE relationship_id = ? AND execution_generation = ?", rid, number) + if err == nil { + bound = generationRecord(current) + } + return err }) - if err != nil { - return nil, err - } - g, err := one(ctx, s, "SELECT * FROM generations WHERE relationship_id = ? AND execution_generation = ?", rid, number) - return generationRecord(g), err + return bound, err } func generationRecord(g Row) Obj { diff --git a/internal/relay/delivery/anchor_test.go b/internal/relay/delivery/anchor_test.go index 76581389..a0c32e5c 100644 --- a/internal/relay/delivery/anchor_test.go +++ b/internal/relay/delivery/anchor_test.go @@ -1,6 +1,7 @@ package delivery import ( + "sync" "testing" "github.com/thisisjun786/codex-relay-workflow/internal/relay/store" @@ -145,6 +146,63 @@ func TestANB01_every_route_to_dispatched_binds_the_new_anchor(t *testing.T) { }) } +func TestBindAnchor_concurrent_turns_never_replace_the_winner(t *testing.T) { + a := newANB(t, t.TempDir()) + a.revisionPending() + var start sync.WaitGroup + start.Add(1) + type outcome struct { + turn string + row Obj + err error + } + results := make(chan outcome, 2) + for _, turn := range []string{"turn-A", "turn-B"} { + go func(turn string) { + start.Wait() + row, err := BindAnchor(a.ctx, a.store, a.clock, a.rid, 2, turn) + results <- outcome{turn, row, err} + }(turn) + } + start.Done() + first, second := <-results, <-results + var winner, loser outcome + if first.err == nil { + winner, loser = first, second + } else { + winner, loser = second, first + } + if winner.err != nil || Reason(loser.err) != "anchor_already_bound" || + str(winner.row, "dispatchTurnId") != winner.turn || + str(a.gen2(), "dispatchTurnId") != winner.turn { + t.Fatalf("winner=%+v loser=%+v generation=%v", winner, loser, a.gen2()) + } + if _, err := BindAnchor(a.ctx, a.store, a.clock, a.rid, 2, winner.turn); err != nil { + t.Fatalf("same-turn replay: %v", err) + } + if a.one("SELECT count(*) AS n FROM journal WHERE kind = 'anchor_bound'").I("n") != 1 { + t.Fatal("binding journal must be written once") + } +} + +func TestBindPendingAnchors_recovers_acknowledged_revision(t *testing.T) { + a := newANB(t, t.TempDir()) + rev := a.dispatchRevision("") + a.clock.Advance(3600) + a.mustAttempt(rev, at(a.clock.Now())) + if str(a.gen2(), "anchorState") != "anchor_pending" { + t.Fatalf("expected pending generation: %v", a.gen2()) + } + _, err := execSQL(a.ctx, a.store, "UPDATE deliveries SET state = ? WHERE event_id = ?", Acknowledged, rev) + mustDo(t, err) + // Python ack.bind_dispatched_revision rejects ACKNOWLEDGED despite selecting it + // in bind_pending_anchors. Recovery must bind it rather than pin that defect. + bound := a.bindPending() + if len(bound) != 1 || bound[0] != rev || str(a.gen2(), "anchorState") != "bound" || len(a.bindPending()) != 0 { + t.Fatalf("bound=%v generation=%v", bound, a.gen2()) + } +} + func TestANB02_an_unbound_generation_refuses_the_childs_receipt(t *testing.T) { runANB(t, "unbound", func(a *anb, out map[string]any) { rev := a.dispatchRevision("") diff --git a/internal/relay/delivery/cli.go b/internal/relay/delivery/cli.go index 635cd33c..c89ebe7e 100644 --- a/internal/relay/delivery/cli.go +++ b/internal/relay/delivery/cli.go @@ -225,6 +225,16 @@ func ExecuteAs(ctx context.Context, prog string, argv []string, stdout, stderr i return 2, true } run := &cliRun{ctx: ctx, args: parsed, socket: socket, clock: SystemClock{}} + if command == "ack-proof" && check != nil { + if err := check(store.StateSelection{}, socket); err != nil { + var payload PayloadError + if errors.As(err, &payload) { + body, code := payload.ExitPayload() + return reply(stdout, body, code), true + } + return reply(stdout, Obj{{Key: "error", Value: "host"}, {Key: "detail", Value: hostDetail(err)}}, contract.ExitHost), true + } + } if command != "ack-proof" { selection, err := store.ResolveStateDir(state, socket) if err != nil { From 5a8d455a02507813e0519a1142d1a9825a6e6d87 Mon Sep 17 00:00:00 2001 From: thisisjun786 <259586770+thisisjun786@users.noreply.github.com> Date: Sun, 27 Sep 2026 00:50:30 +0900 Subject: [PATCH 3/3] fix(relay/delivery): reconcile gate fingerprint includes the receipt turn id --- docs/port/known-defects.md | 2 + .../relay/delivery/hostloss_harness_test.go | 16 ++++++++ .../relay/delivery/reconcile_gate_test.go | 40 +++++++++++++++++++ internal/relay/delivery/reconcile_pass.go | 9 ++++- 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 internal/relay/delivery/reconcile_gate_test.go diff --git a/docs/port/known-defects.md b/docs/port/known-defects.md index fe6142f7..5cee144f 100644 --- a/docs/port/known-defects.md +++ b/docs/port/known-defects.md @@ -1,4 +1,6 @@ # Python defects not carried over +- **Python defect not carried over:** `daemon._gate` fingerprint omits the receipt turn id (`daemon.py:~993`), so an accepted receipt that gains a turn id later is never reconciled; Go includes it (`TestReconcilePass_receipt_gains_turn_id`). + - **Python defect not carried over:** `registry.bind_anchor` (`registry.py:790-822`) reads the pending anchor before beginning its transaction. Concurrent binders can overwrite a generation's dispatch turn after one has already succeeded, violating the never-rebind invariant. Go reads and decides inside the write transaction; `TestBindAnchor_concurrent_turns_never_replace_the_winner` proves only one distinct turn binds and the loser is refused. - **Python defect not carried over:** `ack.bind_dispatched_revision` (`ack.py:1026-1035`) rejects acknowledged revisions even though `ack.bind_pending_anchors` (`ack.py:1037-1060`) selects them for recovery. This violates the invariant that every dispatched revision's pending anchor can recover after acknowledgement. Go accepts both states; `TestBindPendingAnchors_recovers_acknowledged_revision` proves the recovery binds and is idempotent. diff --git a/internal/relay/delivery/hostloss_harness_test.go b/internal/relay/delivery/hostloss_harness_test.go index 538eece6..c0a4f647 100644 --- a/internal/relay/delivery/hostloss_harness_test.go +++ b/internal/relay/delivery/hostloss_harness_test.go @@ -162,6 +162,22 @@ func requireSameDeliveryTables(t *testing.T, f *fixture, python pyCapture) { got := normalizeJSON(t, f.tables()).(map[string]any) want := normalizeJSON(t, python.Tables).(map[string]any) for _, n := range deliveryTables { + if n == "reconcile_gate" && got[n] != nil { + // Python's gate omits the receipt turn id. Compare its other persisted fields + // unchanged while the Go-only regression tests verify the extra component. + for _, value := range got[n].([]any) { + row := value.(map[string]any) + if fingerprint, ok := row["fingerprint"].(string); ok { + status, tail, found := strings.Cut(fingerprint, "|") + if found { + _, content, found := strings.Cut(tail, "|") + if found { + row["fingerprint"] = status + "|" + content + } + } + } + } + } if !reflect.DeepEqual(got[n], want[n]) { g, _ := json.MarshalIndent(got[n], "", " ") w, _ := json.MarshalIndent(want[n], "", " ") diff --git a/internal/relay/delivery/reconcile_gate_test.go b/internal/relay/delivery/reconcile_gate_test.go new file mode 100644 index 00000000..7ab116bd --- /dev/null +++ b/internal/relay/delivery/reconcile_gate_test.go @@ -0,0 +1,40 @@ +package delivery + +import ( + "strings" + "testing" +) + +func TestReconcilePass_receipt_gains_turn_id(t *testing.T) { + f := newFixture(t, "") + event := f.queuedEvent(regOpts{}) + f.host.script = []string{"in_progress"} + request := str(f.mustAttempt(event, nil), "requestId") + f.host.ledger[request] = Obj{{Key: "status", Value: Accepted}} + f.clock.Advance(1000) + rc := NewReconciler(f.delivery) + pass := func() ReconcileReport { + t.Helper() + var report ReconcileReport + mustDo(t, ReconcilePass(f.ctx, rc, f.host, 8, f.clock.Now(), &report)) + return report + } + if report := pass(); report.Reconciled != 1 || f.row(event).S("state") != HeldUncertain { + t.Fatalf("accepted receipt without turn must remain held: report=%+v delivery=%v", report, f.row(event)) + } + if report := pass(); report.Reconciled != 0 || report.Skipped != 1 { + t.Fatalf("unchanged receipt must skip: %+v", report) + } + fingerprint := f.one("SELECT fingerprint FROM reconcile_gate WHERE request_id = ?", request).S("fingerprint") + f.host.ledger[request] = Obj{{Key: "status", Value: Accepted}, {Key: "turnId", Value: "turn-42"}} + if report := pass(); report.Reconciled != 1 || report.Skipped != 0 { + t.Fatalf("receipt with new turn must reconcile: %+v", report) + } + row := f.row(event) + if row.S("state") != Dispatched || row.S("dispatch_turn_id") != "turn-42" { + t.Fatalf("receipt turn must dispatch delivery: %v", row) + } + if got := f.one("SELECT fingerprint FROM reconcile_gate WHERE request_id = ?", request).S("fingerprint"); got == fingerprint || !strings.Contains(got, "turn-42") { + t.Fatalf("fingerprint must track the turn id: before=%q after=%q", fingerprint, got) + } +} diff --git a/internal/relay/delivery/reconcile_pass.go b/internal/relay/delivery/reconcile_pass.go index 0152adbc..fb5913d8 100644 --- a/internal/relay/delivery/reconcile_pass.go +++ b/internal/relay/delivery/reconcile_pass.go @@ -216,13 +216,18 @@ func gate(ctx context.Context, rc *Reconciler, adapter Adapter, attempt Row) (bo if readErr != nil { return true, nil, markGate(ctx, rc, id, nil, true, readErr.Error()) } - status := "missing" + status, turnID := "missing", "" if receipt != nil { if v, ok := get(receipt, "status"); ok { status = pyStr(v) } + if v, ok := get(receipt, "turnId"); ok { + if turn, ok := usableTurnID(v).(string); ok { + turnID = turn + } + } } - fingerprint := status + "|" + content + fingerprint := status + "|" + turnID + "|" + content if row == nil || row.I("retry_required") != 0 { return true, fingerprint, nil }