diff --git a/README.md b/README.md index 321b93d..5dbd1ac 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,22 @@ subtest scheduling, and do not use a parent's `defer` to release resources neede by native parallel children. Cleanup waits for those children in native runs. These callbacks cannot recover resources after process termination; persistent fixture providers must retain ownership and support external reconciliation. + +## Skipped prerequisites + +`testy.Skipf(t, "reason: %s", detail)` stops a case whose prerequisites cannot +be established, while running defers and registered cleanup. Use only at explicit +prerequisite boundaries, not to suppress product assertion failures. The native +runner uses Go's skipped status; hosted results retain `SkipReason`, including +when a prior or later failure overrides the skip. `testy.Skipped(t)` lets cleanup +adapters identify an explicit skip; it does not mean that the test has not failed. +These optional capabilities do not add methods to `TestingT`. + +Hosted skipped leaves are `skipped`; nonfailed containers with skipped coverage +are `incomplete`, never `passed`. `SumTestStatsWithSkipped` returns total, passed, +failed and skipped counts. The existing `SumTestStats` signature is preserved, +but its total includes skipped tests without counting them as passed. Result +consumers must handle incomplete coverage explicitly before enabling skips. +Any `Fail`, `Errorf`, fatal error or panic, including during cleanup, remains a +failure. A prerequisite-specific best-effort cleanup policy must report warnings +explicitly instead of weakening these generic failure semantics. diff --git a/db.go b/db.go index 7dbab66..949f9b8 100644 --- a/db.go +++ b/db.go @@ -41,6 +41,8 @@ type Summary struct { Passed int // Failed is the number of tests that failed. Failed int + // Skipped is unexecuted coverage, not included in Passed. + Skipped int } // TruncatedTimestamp returns the started timestamp truncated to second precision. @@ -87,7 +89,7 @@ var _ DB = (*InMemoryDB)(nil) func (db *InMemoryDB) Enumerate(_ context.Context, _ int) (results []Summary, more bool, err error) { s := make([]Summary, 0, len(db.store)) db.store.Iterate(func(id string, r TestResult) bool { - total, passed, failed := r.SumTestStats() + total, passed, failed, skipped := r.SumTestStatsWithSkipped() s = append(s, Summary{ ID: id, Started: r.Started, @@ -95,6 +97,7 @@ func (db *InMemoryDB) Enumerate(_ context.Context, _ int) (results []Summary, mo Total: total, Passed: passed, Failed: failed, + Skipped: skipped, }) return true }) diff --git a/run.go b/run.go index d797c19..c0792ba 100644 --- a/run.go +++ b/run.go @@ -72,7 +72,7 @@ func RunAsTest(t *testing.T) { pkgTests.BeforeTest(tWrapper{t: tt}) } - test.tester(newTWrapper(tt, context.Background())) + newTWrapper(tt, context.Background()).run(test.tester) }) return true }) @@ -173,10 +173,7 @@ func BuildSuiteResult(start time.Time, packageResults []TestResult) TestResult { r := ResultPassed for _, pkgResult := range packageResults { - if pkgResult.Result == ResultFailed { - r = ResultFailed - break - } + r = combineResults(r, pkgResult.Result) } results.Result = r dur := time.Since(start).Round(time.Millisecond) @@ -360,6 +357,9 @@ func runPackage(pkg string, pkgTests *testPkg) TestResult { } r := ResultPassed + for _, child := range pkgResults.Subtests { + r = combineResults(r, child.Result) + } if pkgAnyFailures { r = ResultFailed } @@ -426,9 +426,19 @@ func runTestContext(ctx context.Context, pkg, baseName string, tester Tester) Te dur := time.Since(start).Round(time.Millisecond) r := ResultPassed + for _, child := range result.Subtests { + r = combineResults(r, child.Result) + } + if t.skipped { + r = ResultSkipped + if len(result.Subtests) > 0 { + r = ResultIncomplete + } + } if t.failed || anyFailures { r = ResultFailed } + result.SkipReason = t.skipReason result.Msgs = t.msgs result.Result = r result.Started = start diff --git a/skip.go b/skip.go new file mode 100644 index 0000000..299fbdf --- /dev/null +++ b/skip.go @@ -0,0 +1,29 @@ +package testy + +// Skipf records why this test cannot execute and stops it with runtime.Goexit, +// running its defers and registered cleanup. A prior or later failure always +// overrides the skipped outcome. Call only from the test's goroutine. +// +// Skipping is an optional capability, preserving existing TestingT implementations. +// Custom runners must implement Skipf(string, ...interface{}); unsupported runners +// panic rather than silently report unexecuted assertions as passed. Legacy +// Before/After hooks are not test scopes and cannot skip. +func Skipf(t TestingT, format string, args ...interface{}) { + t.Helper() + skipper, ok := t.(interface{ Skipf(string, ...interface{}) }) + if !ok { + panic("testy: TestingT does not support Skipf") + } + skipper.Skipf(format, args...) +} + +// Skipped reports whether the test requested a skip, including during cleanup. +// This does not mean it passed or cannot also have failed. Custom runners must +// implement Skipped() bool to use this optional capability. +func Skipped(t TestingT) bool { + skipper, ok := t.(interface{ Skipped() bool }) + if !ok { + panic("testy: TestingT does not support Skipped") + } + return skipper.Skipped() +} diff --git a/skip_test.go b/skip_test.go new file mode 100644 index 0000000..ae02b68 --- /dev/null +++ b/skip_test.go @@ -0,0 +1,222 @@ +package testy + +import ( + "context" + "os" + "os/exec" + "reflect" + "strings" + "testing" + "time" + + "github.com/gametimesf/testy/internal/orderedmap" +) + +func exerciseSkip(tt TestingT, mode string, record func(string)) { + ctx := Context(tt) + Cleanup(tt, func() { record("first") }) + Cleanup(tt, func() { + record("last") + if ctx.Err() != context.Canceled { + tt.Errorf("cleanup context not canceled") + } + if !Skipped(tt) { + tt.Errorf("skip not visible during cleanup") + } + switch mode { + case "cleanup-error": + tt.Errorf("cleanup error") + case "cleanup-fatal": + tt.Fatal("cleanup fatal") + case "cleanup-panic": + panic("cleanup panic") + } + }) + defer record("defer") + switch mode { + case "prior-failure": + tt.Fail() + case "deferred-failure": + defer tt.Errorf("deferred failure") + } + Skipf(tt, "setup unavailable: %s", "fixture") + record("unreachable") +} + +func TestHostedSkipLifecycle(t *testing.T) { + for _, mode := range []string{"skip", "prior-failure", "deferred-failure", "cleanup-error", "cleanup-fatal", "cleanup-panic"} { + t.Run(mode, func(t *testing.T) { + var order []string + result := runTest("skip", mode, func(tt TestingT) { exerciseSkip(tt, mode, func(s string) { order = append(order, s) }) }) + want := ResultSkipped + if mode != "skip" { + want = ResultFailed + } + if result.Result != want || result.SkipReason != "setup unavailable: fixture" { + t.Fatalf("result: %+v", result) + } + if !reflect.DeepEqual(order, []string{"defer", "last", "first"}) { + t.Fatalf("order %v", order) + } + }) + } +} + +func TestNativeSkipLifecycle(t *testing.T) { + if mode := os.Getenv("TESTY_SKIP_MODE"); mode != "" { + exerciseSkip(newTWrapper(t, context.Background()), mode, func(s string) { t.Log("skip-marker:" + s) }) + return + } + for _, mode := range []string{"skip", "prior-failure", "deferred-failure", "cleanup-error", "cleanup-fatal", "cleanup-panic"} { + t.Run(mode, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestNativeSkipLifecycle$", "-test.v") + cmd.Env = append(os.Environ(), "TESTY_SKIP_MODE="+mode) + output, err := cmd.CombinedOutput() + if (err != nil) != (mode != "skip") { + t.Fatalf("error %v: %s", err, output) + } + remaining := string(output) + for _, marker := range []string{"defer", "last", "first"} { + i := strings.Index(remaining, "skip-marker:"+marker) + if i < 0 { + t.Fatalf("missing marker %s: %s", marker, output) + } + remaining = remaining[i+len("skip-marker:"+marker):] + } + if mode == "skip" && !strings.Contains(string(output), "--- SKIP:") { + t.Fatalf("not skipped: %s", output) + } + }) + } +} + +func TestSkipAggregation(t *testing.T) { + for _, tc := range []struct { + name string + cases []Tester + result Result + counts [4]int + }{ + {"all-skipped", []Tester{func(tt TestingT) { Skipf(tt, "setup") }}, ResultIncomplete, [4]int{1, 0, 0, 1}}, + {"mixed", []Tester{func(TestingT) {}, func(tt TestingT) { Skipf(tt, "setup") }}, ResultIncomplete, [4]int{2, 1, 0, 1}}, + {"failure", []Tester{func(tt TestingT) { tt.Fail() }, func(tt TestingT) { Skipf(tt, "setup") }}, ResultFailed, [4]int{2, 0, 1, 1}}, + } { + t.Run(tc.name, func(t *testing.T) { + pkg := &testPkg{tests: orderedmap.OrderedMap[string, testCase]{}} + for i, tester := range tc.cases { + name := string(rune('a' + i)) + pkg.tests[name] = testCase{Name: name, tester: tester} + } + result := BuildSuiteResult(time.Now(), []TestResult{runPackage("skip", pkg)}) + if result.Result != tc.result || result.Subtests[0].Result != tc.result { + t.Fatalf("result %+v", result) + } + total, passed, failed, skipped := result.SumTestStatsWithSkipped() + if got := [4]int{total, passed, failed, skipped}; got != tc.counts { + t.Fatalf("counts %v want %v", got, tc.counts) + } + a, b, c := result.SumTestStats() + if [3]int{a, b, c} != [3]int{total, passed, failed} { + t.Fatal("legacy stats disagree") + } + db := &InMemoryDB{} + if _, err := db.Save(context.Background(), result); err != nil { + t.Fatal(err) + } + summaries, _, err := db.Enumerate(context.Background(), 1) + if err != nil || summaries[0].Skipped != skipped { + t.Fatalf("summaries %+v err %v", summaries, err) + } + }) + } +} + +func TestSkipNestedResult(t *testing.T) { + result := runTest("skip", "parent", func(tt TestingT) { + if !tt.Run("child", func(child TestingT) { Skipf(child, "missing fixture") }) { + tt.Errorf("skip is not failure") + } + }) + if result.Result != ResultIncomplete { + t.Fatalf("result %+v", result) + } + parentSkip := runTest("skip", "parent", func(tt TestingT) { tt.Run("child", func(TestingT) {}); Skipf(tt, "remaining setup") }) + if parentSkip.Result != ResultIncomplete { + t.Fatalf("parent not incomplete: %+v", parentSkip) + } + total, passed, failed, skipped := parentSkip.SumTestStatsWithSkipped() + if [4]int{total, passed, failed, skipped} != [4]int{2, 1, 0, 1} { + t.Fatalf("parent skip not counted: %+v", parentSkip) + } +} + +func TestSkipCannotMaskPanic(t *testing.T) { + for _, bodyPanic := range []bool{true, false} { + result := runTest("skip", "panic", func(tt TestingT) { + Cleanup(tt, func() { Skipf(tt, "cleanup skip") }) + if bodyPanic { + panic("body panic") + } + Cleanup(tt, func() { panic("cleanup panic") }) + }) + if result.Result != ResultFailed { + t.Fatalf("panic masked: %+v", result) + } + } +} + +func TestSkipUnsupported(t *testing.T) { + for _, f := range []func(TestingT){func(tt TestingT) { Skipf(tt, "unsupported") }, func(tt TestingT) { Skipped(tt) }} { + func() { + defer func() { + if recover() == nil { + t.Error("unsupported must panic") + } + }() + f(&tHelper{}) + }() + } +} + +func TestNativeSkipCannotMaskCleanupPanic(t *testing.T) { + if os.Getenv("TESTY_SKIP_PANIC") != "" { + tt := newTWrapper(t, context.Background()) + Cleanup(tt, func() { Skipf(tt, "cleanup skip") }) + Cleanup(tt, func() { panic("real cleanup defect") }) + return + } + cmd := exec.Command(os.Args[0], "-test.run=^TestNativeSkipCannotMaskCleanupPanic$", "-test.v") + cmd.Env = append(os.Environ(), "TESTY_SKIP_PANIC=1") + output, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(output), "real cleanup defect") { + t.Fatalf("panic masked: err=%v output=%s", err, output) + } +} + +func TestNativeSkipCannotMaskBodyPanic(t *testing.T) { + if mode := os.Getenv("TESTY_BODY_PANIC"); mode != "" { + body := func(tt TestingT) { + Cleanup(tt, func() { Skipf(tt, "cleanup skip") }) + panic("real body defect") + } + pkg := &testPkg{tests: orderedmap.OrderedMap[string, testCase]{}} + pkg.tests["case"] = testCase{Name: "case", tester: func(tt TestingT) { + if mode == "nested" { + tt.Run("child", body) + } else { + body(tt) + } + }} + instance = testy{tests: orderedmap.OrderedMap[string, *testPkg]{"skip": pkg}} + RunAsTest(t) + return + } + for _, mode := range []string{"case", "nested"} { + cmd := exec.Command(os.Args[0], "-test.run=^TestNativeSkipCannotMaskBodyPanic$", "-test.v") + cmd.Env = append(os.Environ(), "TESTY_BODY_PANIC="+mode) + output, err := cmd.CombinedOutput() + if err == nil || !strings.Contains(string(output), "real body defect") { + t.Fatalf("panic masked mode=%s err=%v output=%s", mode, err, output) + } + } +} diff --git a/t.go b/t.go index ced7241..3f8ae6f 100644 --- a/t.go +++ b/t.go @@ -12,6 +12,8 @@ type t struct { name string tester Tester failed bool + skipped bool + skipReason string msgs []Msg subtests chan<- subtest subtestDone <-chan bool @@ -41,6 +43,8 @@ func (t *t) run() { }() defer t.lifecycle.finish() + // Record a body panic before cleanup: a later Goexit must not mask it. + defer t.recoverPanic() t.tester(t) } @@ -103,7 +107,15 @@ func (t *t) Cleanup(f func()) { if t.lifecycle == nil { panic("testy: Cleanup is only supported in Test and Run callbacks") } - t.lifecycle.add(f) + if f == nil { + panic("testy: nil cleanup") + } + t.lifecycle.add(func() { + // Record each panic before proceeding to the remaining callbacks, which + // may themselves call FailNow or Skipf (Goexit). + defer t.recoverPanic() + f() + }) } func (t *t) Context() context.Context { @@ -112,3 +124,21 @@ func (t *t) Context() context.Context { } return t.lifecycle.ctx } + +func (t *t) recoverPanic() { + if err := recover(); err != nil { + t.Errorf("panic: %+v", err) + } +} + +func (t *t) Skipf(format string, args ...interface{}) { + if !t.test() { + panic("testy: Skipf is only supported in Test and Run callbacks") + } + t.skipped = true + t.skipReason = fmt.Sprintf(format, args...) + t.Logf("skipped: %s", t.skipReason) + runtime.Goexit() +} + +func (t *t) Skipped() bool { return t.skipped } diff --git a/templates/result.gohtml b/templates/result.gohtml index 0ef78b7..fd2d841 100644 --- a/templates/result.gohtml +++ b/templates/result.gohtml @@ -1,5 +1,5 @@ {{define "singleResult"}} - + {{- /*gotype: github.com/gametimesf/testy.TestResult*/ -}} {{.Package}} {{.Name}} @@ -7,7 +7,7 @@ {{.DurHuman}} {{.Result}} - {{.PassedSubtests}} / {{.FailedSubtests}} / {{.TotalSubtests}} + {{.PassedSubtests}} / {{.FailedSubtests}} / {{.SkippedSubtests}} / {{.TotalSubtests}} {{if .Msgs}} @@ -58,13 +58,13 @@ Started At Duration Result - Subtest Results (Passed / Failed / Total) + Subtest Results (Passed / Failed / Skipped / Total) Messages {{with .Result}} - + {{- /*gotype: github.com/gametimesf/testy.TestResult*/ -}} {{.Name}} @@ -72,7 +72,7 @@ {{.DurHuman}} {{.Result}} - {{.PassedSubtests}} / {{.FailedSubtests}} / {{.TotalSubtests}} + {{.PassedSubtests}} / {{.FailedSubtests}} / {{.SkippedSubtests}} / {{.TotalSubtests}} diff --git a/templates/result_list.gohtml b/templates/result_list.gohtml index cda3b4e..c6c47b5 100644 --- a/templates/result_list.gohtml +++ b/templates/result_list.gohtml @@ -13,20 +13,22 @@ Start Time Duration - Total Tests Executed + Total Tests Tests Passed Tests Failed + Tests Skipped {{- /*gotype: github.com/gametimesf/testy.listResultsCtx*/ -}} {{range .Results}} - + {{.TruncatedTimestamp}} {{.Dur}} {{.Total}} {{.Passed}} {{.Failed}} + {{.Skipped}} {{end}} diff --git a/testy.go b/testy.go index dc09298..6654fe6 100644 --- a/testy.go +++ b/testy.go @@ -40,6 +40,8 @@ type TestResult struct { Msgs []Msg // Result is the result of the test. Result Result + // SkipReason records an explicit skip request, even if a later failure wins. + SkipReason string `json:",omitempty"` // Started is when the test was started. Started time.Time // Dur is how long the test took. @@ -68,6 +70,10 @@ const ( ResultPassed Result = "passed" // ResultFailed indicates that this test or at least one of its subtests failed. ResultFailed Result = "failed" + // ResultSkipped indicates this test explicitly skipped execution. It is not a pass. + ResultSkipped Result = "skipped" + // ResultIncomplete indicates a nonfailed container with skipped coverage. + ResultIncomplete Result = "incomplete" ) type Msg struct { @@ -140,31 +146,60 @@ func sanitizeName(r rune) rune { } } -// SumTestStats returns the total number of leaf subtests, as well as the number of those that passed and failed. +// SumTestStats returns total, passed and failed counts. Skipped tests are in +// total but NOT passed; use SumTestStatsWithSkipped for complete coverage counts. func (tr TestResult) SumTestStats() (total, passed, failed int) { + total, passed, failed, _ = tr.SumTestStatsWithSkipped() + return +} + +// SumTestStatsWithSkipped counts leaf outcomes and parent-only failures/skips. +// Unknown leaf outcomes are conservatively counted as failed, never passed. +func (tr TestResult) SumTestStatsWithSkipped() (total, passed, failed, skipped int) { if len(tr.Subtests) == 0 { - if tr.Result == ResultFailed { - return 1, 0, 1 - } else { - return 1, 1, 0 + switch tr.Result { + case ResultPassed: + return 1, 1, 0, 0 + case ResultSkipped, ResultIncomplete: + return 1, 0, 0, 1 + default: + return 1, 0, 1, 0 } } - for _, st := range tr.Subtests { - t, p, f := st.SumTestStats() + t, p, f, s := st.SumTestStatsWithSkipped() total += t passed += p failed += f + skipped += s } if tr.Result == ResultFailed && failed == 0 { - // The node itself failed even though every leaf under it passed — - // e.g. an error raised in the parent test body or its cleanup after - // subtests completed. Count it so totals cannot report 100% passed - // for a failed tree. + // Attribute a body/cleanup failure even if all children passed or skipped. total++ failed++ + } else if (tr.Result == ResultSkipped || tr.Result == ResultIncomplete) && skipped == 0 { + // The parent skipped remaining work after its children completed. + total++ + skipped++ + } + return +} + +// SkippedSubtests returns skipped coverage, including a parent-only skip. +func (tr TestResult) SkippedSubtests() int { + _, _, _, skipped := tr.SumTestStatsWithSkipped() + return skipped +} + +// combineResults preserves failure precedence and exposes incomplete coverage. +func combineResults(current, next Result) Result { + if current == ResultFailed || next == ResultFailed { + return ResultFailed + } + if current != ResultPassed || next != ResultPassed { + return ResultIncomplete } - return total, passed, failed + return ResultPassed } // TotalSubtests returns the total number of leaf subtests. diff --git a/twrapper.go b/twrapper.go index 457e657..a2e4bdc 100644 --- a/twrapper.go +++ b/twrapper.go @@ -61,7 +61,7 @@ func (t tWrapper) Run(s string, tester Tester) bool { if t.lifecycle != nil { parent = t.lifecycle.ctx } - tester(newTWrapper(tt, parent)) + newTWrapper(tt, parent).run(tester) }) } @@ -85,7 +85,19 @@ func (t tWrapper) Cleanup(f func()) { if t.lifecycle == nil { panic("testy: Cleanup is only supported in Test and Run callbacks") } - t.lifecycle.add(f) + if f == nil { + panic("testy: nil cleanup") + } + t.lifecycle.add(func() { + // Mark the failure before another callback can mask this panic with Goexit. + defer func() { + if err := recover(); err != nil { + t.t.Errorf("panic: %+v", err) + panic(err) + } + }() + f() + }) } func (t tWrapper) Context() context.Context { @@ -94,3 +106,23 @@ func (t tWrapper) Context() context.Context { } return t.lifecycle.ctx } + +func (t tWrapper) Skipf(format string, args ...interface{}) { + if t.lifecycle == nil { + panic("testy: Skipf is only supported in Test and Run callbacks") + } + t.t.Helper() + t.t.Skipf(format, args...) +} + +func (t tWrapper) Skipped() bool { return t.t.Skipped() } + +func (t tWrapper) run(tester Tester) { + defer func() { + if err := recover(); err != nil { + t.t.Errorf("panic: %+v", err) + panic(err) + } + }() + tester(t) +}