From 328a4561d57126500f2adcd85d7d4498da2a44c5 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Sun, 18 Oct 2026 20:35:51 +0200 Subject: [PATCH 1/4] Exit cleanly after test failures --- .github/workflows/ci.yml | 13 ------------- README.md | 2 +- scripts/check-output.py | 25 +++++++++++++++++++++++-- src/python/Host.jo | 4 ++++ src/ruby/Host.jo | 4 ++++ src/runner/Runner.jo | 4 ++-- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fada2c6..910e89b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,19 +31,6 @@ jobs: - run: jo run test-python - run: jo run test-ruby - run: python3 scripts/check-output.py - - name: Check runner failure exit status - run: | - for module in test-python-failure test-ruby-failure; do - output=$(mktemp) - if jo run "$module" >"$output" 2>&1; then - cat "$output" - echo "$module unexpectedly succeeded" - exit 1 - fi - grep -q 'tests failed' "$output" - cat "$output" - rm "$output" - done - name: Build release artifacts run: | jo package core diff --git a/README.md b/README.md index 9a19f46..4947f0b 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ jo run tests -- "math/adds numbers" Assertions accumulate failures and retain their source locations. Unexpected exceptions fail the current test, and execution continues. `Runner.runWithArgs` -exits with a non-zero status if any selected test fails. Use `Runner.run` when +exits with status 1 if any selected test fails. Use `Runner.run` when you want to handle its Boolean result yourself. Filters are test-path prefixes. The root's name is not part of a path. A filter diff --git a/scripts/check-output.py b/scripts/check-output.py index b9d2f42..9c0ee52 100644 --- a/scripts/check-output.py +++ b/scripts/check-output.py @@ -1,4 +1,4 @@ -"""Check observable reporting parity after building the conformance apps.""" +"""Check reporting parity and clean failure exits on both backends.""" import re import subprocess @@ -20,7 +20,28 @@ def report(platform): return re.sub(r" in [0-9.]+s", " in s", output) +def failure_report(platform): + result = subprocess.run( + ["jo", "run", f"test-{platform}-failure"], capture_output=True, text=True + ) + output = result.stdout + result.stderr + assert result.returncode == 1, (result.returncode, output) + assert ( + "intentional failure: intentional (test-fixtures/failure/Run.jo:" + in result.stdout + ), output + assert "0 passed, 1 failed" in result.stdout, output + assert "Traceback" not in output, output + assert "SystemExit" not in output, output + assert "tests failed" not in output, output + assert "src/runner/Runner.jo" not in output, output + return re.sub(r" in [0-9.]+s", " in s", result.stdout) + + if __name__ == "__main__": assert Path("jo.toml").exists(), "run from the repository root" assert report("python") == report("ruby"), "backend reports differ" - print("Python and Ruby reports match") + assert failure_report("python") == failure_report("ruby"), ( + "backend failure reports differ" + ) + print("Python and Ruby reports match; failure exits are clean") diff --git a/src/python/Host.jo b/src/python/Host.jo index a339b95..75c0103 100644 --- a/src/python/Host.jo +++ b/src/python/Host.jo @@ -45,6 +45,10 @@ private def hostFlush(): Unit = py.module("sys").stdout.flush() pass +private def hostExit(status: Int): Unit = + py.module("sys").exit(status) + pass + private def hostTry(body: () => Unit): Result[Bool, String] = match py.try(body()) case Ok(_) => Ok(true) diff --git a/src/ruby/Host.jo b/src/ruby/Host.jo index 0904def..1718748 100644 --- a/src/ruby/Host.jo +++ b/src/ruby/Host.jo @@ -42,6 +42,10 @@ private def hostFlush(): Unit = rb.const("STDOUT").flush() pass +private def hostExit(status: Int): Unit = + rb.const("Kernel").exit(status) + pass + private def hostTry(body: () => Unit): Result[Bool, String] = match rb.try(body()) case Ok(_) => Ok(true) diff --git a/src/runner/Runner.jo b/src/runner/Runner.jo index 3fc800d..a0f1d35 100644 --- a/src/runner/Runner.jo +++ b/src/runner/Runner.jo @@ -160,12 +160,12 @@ section Runner //[ Run tests using the conventional first-argument filter. ! - ! Aborts with a non-zero process status when any selected test fails. Use + ! Exits with process status 1 when any selected test fails. Use ! `run` when the caller needs to handle the Boolean result itself. //] def runWithArgs(root: Testing.Suite): Unit receives stdout, args = if !run(root, filterFromArgs(args)) then - abort("tests failed") + hostExit(1) // The root contributes no path segment, so a filter reads as `providers/openai`. private def walk( From 2c44316f892676d85a80bbcb5b31cf99ce924b95 Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Sun, 18 Oct 2026 20:37:38 +0200 Subject: [PATCH 2/4] Add pull request checklist and compatibility template --- .github/pull_request_template.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..a69b429 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,22 @@ +## Summary + + + +## Tests + + + +- [ ] `jo run test-python` +- [ ] `jo run test-ruby` +- [ ] `python3 scripts/check-output.py` +- [ ] `jo package core` +- [ ] `jo package python` +- [ ] `jo package ruby` + +## Changelog + +- [ ] Update `CHANGELOG.md`. + +## Compatibility + + From e1cb06025ab80d470728bb729c7a2f7a308c303f Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Sun, 18 Oct 2026 20:39:11 +0200 Subject: [PATCH 3/4] Document clean failure exits in changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 877e25c..03df7c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [Unreleased] + +### Fixed + +- `Runner.runWithArgs` uses normal process exit with status 1 after reporting + failed tests on Python and Ruby, removing the extra runner exception + traceback. ([#2](https://github.com/typescope/jtest/issues/2)) + ## [0.2.0] - 2026-10-13 ### Breaking changes From 07b7b5a7eee34d3a3c09f9e8545604a4ececb74d Mon Sep 17 00:00:00 2001 From: Fengyun Liu Date: Sun, 18 Oct 2026 20:43:39 +0200 Subject: [PATCH 4/4] Simplify pull request checklist --- .github/pull_request_template.md | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index a69b429..8c69a9a 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -2,20 +2,10 @@ -## Tests +## Check list - - -- [ ] `jo run test-python` -- [ ] `jo run test-ruby` -- [ ] `python3 scripts/check-output.py` -- [ ] `jo package core` -- [ ] `jo package python` -- [ ] `jo package ruby` - -## Changelog - -- [ ] Update `CHANGELOG.md`. +- [ ] Add tests +- [ ] Update `CHANGELOG.md` ## Compatibility