From 531b0ac949085baea7721b2d500a6d13f3d46f12 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 11:24:02 +0200 Subject: [PATCH 01/10] [tests] Stream on-device test results to MTP as they finish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MTP test adapter (`Microsoft.Android.Run`) reported results all-or-nothing at the very end of a run: the on-device instrumentation buffered every result and only wrote a TRX + reported `resultsPath` in its final `Finish()` call, and the host adapter blocked on `am instrument -w`, parsed only the final bundle, then published every `TestNodeUpdateMessage` in one batch. If the app process crashed mid-run (e.g. a native SIGSEGV), `Finish()` never ran, so no TRX was written and no `resultsPath` was reported. The host threw "Instrumentation did not report a resultsPath in the bundle." and MTP showed **Zero tests ran** — discarding the hundreds of tests that had actually passed, with no indication a crash occurred. Stream results instead: * Device (`TestInstrumentation.TestListener`): send an enriched `INSTRUMENTATION_STATUS` block per test on start and finish, carrying the class/name/outcome and (Base64-encoded, so they stay single-line) failure message and stack trace. * Host (`AndroidTestAdapter`): run `am instrument -w -r`, parse the streamed status blocks line-by-line, and publish each test to MTP as it finishes. A `LineWriter` splits the process output into lines and a `StatusStreamParser` feeds completed blocks to an async consumer via a channel (so the sync read loop never blocks on `PublishAsync`). Now a mid-run crash only loses the in-flight test: every test completed beforehand is already reported. When the run doesn't finish cleanly, a synthetic failed test node is published so the run is clearly marked failed rather than silently empty. The TRX-pull path is kept as a fallback for instrumentation that doesn't stream. Also `[Ignore]` the `AndroidMessageHandlerTests .ServerCertificateCustomValidationCallback_*` tests, which crash the test process with a native SIGSEGV (dotnet/android#8608); the root cause will be diagnosed separately. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AndroidTestAdapter.cs | 345 ++++++++++++++++-- .../AndroidMessageHandlerTests.cs | 5 + tests/TestRunner.Core/TestInstrumentation.cs | 45 ++- 3 files changed, 363 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index bd63505f0ab..a34c9332e18 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -1,3 +1,5 @@ +using System.Text; +using System.Threading.Channels; using System.Xml.Linq; using Microsoft.Testing.Extensions.TrxReport.Abstractions; using Microsoft.Testing.Platform.Capabilities.TestFramework; @@ -7,6 +9,7 @@ using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Requests; using Microsoft.Testing.Platform.TestHost; +using Xamarin.Android.Tools; /// /// A Microsoft Testing Platform test framework adapter that runs Android instrumentation @@ -50,9 +53,186 @@ public async Task ExecuteRequestAsync (ExecuteRequestContext context) async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionUid) { - // 1. Run instrumentation on device - var bundleResults = await RunInstrumentationOnDeviceAsync (context.CancellationToken); + // Track tests we've already reported (by their final outcome) as they + // stream in, so we don't double-report them from the pulled TRX. + var reportedFinal = new HashSet (StringComparer.Ordinal); + // 1. Run instrumentation on device, publishing each test to MTP live as + // its INSTRUMENTATION_STATUS block arrives. + var bundleResults = await RunInstrumentationOnDeviceAsync (context, sessionUid, reportedFinal, context.CancellationToken); + + if (verbose) { + Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}, streamed={reportedFinal.Count}"); + if (bundleResults.ResultsPath != null) + Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); + } + + // 2. If we streamed at least one completed test, streaming is authoritative + // (it's resilient to a mid-run crash). Otherwise fall back to the TRX. + if (reportedFinal.Count == 0) { + await ReportFromTrxAsync (context, sessionUid, bundleResults); + return; + } + + // 3. We streamed live results. If the run did not finish cleanly (the app + // process crashed before writing the final results bundle), surface a + // synthetic failed test so the run is clearly marked failed rather than + // silently dropping the in-flight/never-run tests. + if (bundleResults.Crashed) + await PublishCrashNodeAsync (context, sessionUid, bundleResults); + } + + /// + /// Publishes each test to MTP as its INSTRUMENTATION_STATUS block arrives from + /// the streamed 'am instrument -w -r' output, and returns the parsed final + /// results bundle (summary counts, resultsPath, error, crash state). + /// + async Task RunInstrumentationOnDeviceAsync (ExecuteRequestContext context, SessionUid sessionUid, HashSet reportedFinal, CancellationToken cancellationToken) + { + // '-r' prints raw INSTRUMENTATION_STATUS results (so we can stream them); + // '-w' waits for the run to complete. + var cmdArgs = $"shell am instrument -w -r {package}/{instrumentation}"; + var psi = AdbHelper.CreateStartInfo (adbPath, adbTarget, cmdArgs); + + if (verbose) + Console.WriteLine ($"Running: adb {psi.Arguments}"); + + // Completed status blocks are handed off to a single async consumer that + // publishes them to MTP, so the synchronous stdout read loop is never + // blocked on an async PublishAsync. + var channel = Channel.CreateUnbounded (new UnboundedChannelOptions { + SingleReader = true, + SingleWriter = true, + }); + + var fullOutput = new StringBuilder (); + using var stderr = new StringWriter (); + + var consumer = Task.Run (async () => { + await foreach (var status in channel.Reader.ReadAllAsync ()) { + await PublishStatusAsync (context, sessionUid, status, reportedFinal); + } + }); + + var parser = new StatusStreamParser (channel.Writer, fullOutput, verbose); + using var stdout = new LineWriter (parser.OnLine); + + int exitCode; + try { + exitCode = await ProcessUtils.StartProcess (psi, stdout, stderr, cancellationToken); + } finally { + stdout.Flush (); + parser.Complete (); + channel.Writer.Complete (); + } + await consumer; + + var output = fullOutput.ToString (); + if (verbose) { + Console.WriteLine ($"[AndroidTestAdapter] Exit code: {exitCode}"); + Console.WriteLine (output); + } + + var result = ParseInstrumentationBundle (output, stderr.ToString ()); + + // The final bundle (with resultsPath) is only emitted if Finish() ran to + // completion. Its absence — or a non-zero exit / an explicit crash marker — + // means the app process died mid-run. + result.Crashed = + exitCode != 0 || + output.Contains ("INSTRUMENTATION_FAILED", StringComparison.Ordinal) || + output.Contains ("Process crashed", StringComparison.Ordinal) || + (reportedFinal.Count > 0 && result.ResultsPath == null); + + if (result.Crashed && result.Error == null) + result.Error = ExtractCrashMessage (output) ?? (exitCode != 0 ? $"Instrumentation exited with code {exitCode}." : "The test process terminated before reporting a result (likely a native crash)."); + + return result; + } + + /// + /// Publishes a single streamed instrumentation status block to MTP: a "start" + /// event becomes an in-progress node; a "finish" event becomes the final + /// pass/fail/skip node. + /// + async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationStatus status, HashSet reportedFinal) + { + if (!status.Values.TryGetValue ("test", out var fullyQualifiedName) || string.IsNullOrEmpty (fullyQualifiedName)) + return; + + status.Values.TryGetValue ("event", out var eventKind); + status.Values.TryGetValue ("name", out var displayName); + status.Values.TryGetValue ("class", out var className); + + if (eventKind == "start") { + var startNode = new TestNode { + Uid = new TestNodeUid (fullyQualifiedName), + DisplayName = displayName ?? fullyQualifiedName, + Properties = new PropertyBag (new InProgressTestNodeStateProperty ()), + }; + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, startNode)); + return; + } + + status.Values.TryGetValue ("outcome", out var outcome); + var errorMessage = DecodeOrNull (status.Values, "message-b64"); + var stackTrace = DecodeOrNull (status.Values, "stack-b64"); + + var failureMessage = errorMessage ?? "Test failed"; + if (!string.IsNullOrEmpty (stackTrace)) + failureMessage += "\n" + stackTrace; + + var stateProperty = outcome switch { + "passed" => (IProperty) new PassedTestNodeStateProperty (), + "failed" => new FailedTestNodeStateProperty (failureMessage), + "skipped" => new SkippedTestNodeStateProperty (errorMessage), + _ => new PassedTestNodeStateProperty (), + }; + + var properties = new List { stateProperty }; + if (!string.IsNullOrEmpty (className)) + properties.Add (new TrxFullyQualifiedTypeNameProperty (className)); + if (outcome == "failed" && (!string.IsNullOrEmpty (errorMessage) || !string.IsNullOrEmpty (stackTrace))) + properties.Add (new TrxExceptionProperty (errorMessage, stackTrace)); + + var testNode = new TestNode { + Uid = new TestNodeUid (fullyQualifiedName), + DisplayName = displayName ?? fullyQualifiedName, + Properties = new PropertyBag (properties.ToArray ()), + }; + + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, testNode)); + reportedFinal.Add (fullyQualifiedName); + } + + /// + /// Publishes a synthetic failed test node representing a mid-run process crash, + /// so the overall run is reported as failed and the crash is visible in results. + /// + async Task PublishCrashNodeAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults) + { + var message = bundleResults.Error ?? "The test process terminated before all tests completed (likely a native crash)."; + Console.Error.WriteLine ($"[AndroidTestAdapter] {message}"); + + var crashUid = $"{instrumentation}.ProcessCrashed"; + var crashNode = new TestNode { + Uid = new TestNodeUid (crashUid), + DisplayName = "Test process crashed before completion", + Properties = new PropertyBag ( + new FailedTestNodeStateProperty (message), + new TrxFullyQualifiedTypeNameProperty (instrumentation), + new TrxExceptionProperty (message, null)), + }; + + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, crashNode)); + } + + /// + /// Fallback path used only when no results were streamed (e.g. an older + /// on-device instrumentation): pull and parse the TRX, then publish all tests. + /// + async Task ReportFromTrxAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults) + { if (bundleResults.Error != null) { var message = $"Error from instrumentation: {bundleResults.Error}"; if (bundleResults.InstrumentationCode.HasValue) @@ -61,13 +241,6 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU throw new InvalidOperationException (message); } - if (verbose) { - Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}"); - if (bundleResults.ResultsPath != null) - Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); - } - - // 2. Pull and parse TRX if (bundleResults.ResultsPath == null) throw new InvalidOperationException ("Instrumentation did not report a resultsPath in the bundle."); @@ -107,27 +280,34 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU } } - async Task RunInstrumentationOnDeviceAsync (CancellationToken cancellationToken) + static string? DecodeOrNull (IReadOnlyDictionary values, string key) { - var cmdArgs = $"shell am instrument -w {package}/{instrumentation}"; - var (exitCode, output, error) = await AdbHelper.RunAsync (adbPath, adbTarget, cmdArgs, cancellationToken, verbose); - - if (verbose) { - Console.WriteLine ($"[AndroidTestAdapter] Exit code: {exitCode}"); - Console.WriteLine (output); + if (!values.TryGetValue (key, out var encoded) || string.IsNullOrEmpty (encoded)) + return null; + try { + return Encoding.UTF8.GetString (Convert.FromBase64String (encoded)); + } catch (FormatException) { + return encoded; } + } - if (exitCode != 0) { - var failureMessage = !string.IsNullOrWhiteSpace (error) ? error : output; - if (string.IsNullOrWhiteSpace (failureMessage)) - failureMessage = $"adb shell am instrument failed with exit code {exitCode}."; - Console.Error.WriteLine ($"[AndroidTestAdapter] {failureMessage}"); - return new InstrumentationBundleResult { - Error = failureMessage, - }; + /// + /// Extracts a human-readable crash reason from the instrumentation output + /// (shortMsg/longMsg are emitted by 'am instrument' when the process crashes). + /// + static string? ExtractCrashMessage (string output) + { + string? shortMsg = null, longMsg = null; + foreach (var rawLine in output.Split ('\n')) { + var line = rawLine.TrimEnd ('\r'); + if (line.StartsWith ("INSTRUMENTATION_RESULT: shortMsg=", StringComparison.Ordinal)) + shortMsg = line.Substring ("INSTRUMENTATION_RESULT: shortMsg=".Length).Trim (); + else if (line.StartsWith ("INSTRUMENTATION_RESULT: longMsg=", StringComparison.Ordinal)) + longMsg = line.Substring ("INSTRUMENTATION_RESULT: longMsg=".Length).Trim (); } - - return ParseInstrumentationBundle (output, error); + if (longMsg != null || shortMsg != null) + return $"The test process crashed: {longMsg ?? shortMsg}"; + return null; } async Task PullTrxFileAsync (string devicePath, CancellationToken cancellationToken) @@ -282,8 +462,121 @@ class InstrumentationBundleResult public string? ResultsPath { get; set; } public string? Error { get; set; } public int? InstrumentationCode { get; set; } + public bool Crashed { get; set; } } +/// +/// A single completed INSTRUMENTATION_STATUS block: its key/value pairs +/// plus the trailing INSTRUMENTATION_STATUS_CODE. +/// +class InstrumentationStatus +{ + public required IReadOnlyDictionary Values { get; init; } + public int Code { get; init; } +} + +/// +/// A that splits the (arbitrarily chunked) writes it +/// receives from ProcessUtils.StartProcess into complete lines and invokes +/// a callback for each one, so instrumentation output can be parsed as it streams. +/// +sealed class LineWriter (Action onLine) : TextWriter +{ + readonly StringBuilder buffer = new (); + + public override Encoding Encoding => Encoding.UTF8; + + public override void Write (char value) + { + if (value == '\n') + Flush (); + else if (value != '\r') + buffer.Append (value); + } + + public override void Write (char[] buffer, int index, int count) + { + for (int i = 0; i < count; i++) + Write (buffer [index + i]); + } + + public override void Write (string? value) + { + if (value == null) + return; + foreach (var c in value) + Write (c); + } + + // Emit whatever has been buffered as a completed line. + public override void Flush () + { + if (buffer.Length == 0) + return; + var line = buffer.ToString (); + buffer.Clear (); + onLine (line); + } +} + +/// +/// Incrementally parses streamed am instrument -r output. Each completed +/// INSTRUMENTATION_STATUS block (terminated by INSTRUMENTATION_STATUS_CODE) +/// is written to for the consumer to publish, while the +/// raw text is also accumulated for the final results-bundle parse. +/// +sealed class StatusStreamParser (ChannelWriter writer, StringBuilder fullOutput, bool verbose) +{ + const string StatusPrefix = "INSTRUMENTATION_STATUS: "; + const string StatusCodePrefix = "INSTRUMENTATION_STATUS_CODE: "; + + Dictionary? current; + string? lastKey; + + public void OnLine (string line) + { + fullOutput.Append (line).Append ('\n'); + if (verbose) + Console.WriteLine (line); + + if (line.StartsWith (StatusCodePrefix, StringComparison.Ordinal)) { + var codeStr = line.Substring (StatusCodePrefix.Length).Trim (); + int.TryParse (codeStr, out int code); + if (current != null) { + writer.TryWrite (new InstrumentationStatus { Values = current, Code = code }); + current = null; + lastKey = null; + } + return; + } + + if (line.StartsWith (StatusPrefix, StringComparison.Ordinal)) { + var kvp = line.Substring (StatusPrefix.Length); + var eqIndex = kvp.IndexOf ('='); + if (eqIndex > 0) { + current ??= new Dictionary (StringComparer.Ordinal); + lastKey = kvp.Substring (0, eqIndex).Trim (); + current [lastKey] = kvp.Substring (eqIndex + 1); + } + return; + } + + // Continuation of a multi-line value (should be rare — message/stack are + // Base64-encoded on the device precisely to avoid this). + if (current != null && lastKey != null) + current [lastKey] += "\n" + line; + } + + // Drop any block that never received a terminating status code (e.g. the + // process crashed mid-status); it is intentionally not published. + public void Complete () + { + current = null; + lastKey = null; + } +} + + enum TrxOutcome { Passed, diff --git a/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs b/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs index 99cde29dcd3..315bb04bac6 100644 --- a/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs +++ b/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs @@ -137,6 +137,7 @@ public async Task DoesNotDisposeContentStream() } [Test] + [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_ApproveRequest () { bool callbackHasBeenCalled = false; @@ -163,6 +164,7 @@ public async Task ServerCertificateCustomValidationCallback_ApproveRequest () } [Test] + [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_RejectRequest () { bool callbackHasBeenCalled = false; @@ -181,6 +183,7 @@ public async Task ServerCertificateCustomValidationCallback_RejectRequest () } [Test] + [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_ApprovesRequestWithInvalidCertificate () { bool callbackHasBeenCalled = false; @@ -208,6 +211,7 @@ public async Task NoServerCertificateCustomValidationCallback_ThrowsWhenThereIsC } [Test] + [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_IgnoresCertificateHostnameMismatch () { bool callbackHasBeenCalled = false; @@ -229,6 +233,7 @@ public async Task ServerCertificateCustomValidationCallback_IgnoresCertificateHo } [Test] + [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_Redirects () { int callbackCounter = 0; diff --git a/tests/TestRunner.Core/TestInstrumentation.cs b/tests/TestRunner.Core/TestInstrumentation.cs index f9cf928c426..5f849f08ee7 100644 --- a/tests/TestRunner.Core/TestInstrumentation.cs +++ b/tests/TestRunner.Core/TestInstrumentation.cs @@ -311,15 +311,38 @@ static void WriteTrxFile (string path, List assemblyResults) } /// - /// Sends test status updates through the instrumentation protocol. + /// Streams per-test status updates through the instrumentation protocol + /// (am instrument -r) so the host adapter can report each test to MTP + /// as it finishes. This makes results resilient to a mid-run process crash: + /// every test completed before the crash has already been reported, instead + /// of the whole run being lost because the final TRX was never written. + /// + /// Each SendStatus emits an INSTRUMENTATION_STATUS block that + /// the host parses line-by-line. Because that protocol is line-based, the + /// (potentially multi-line) failure message and stack trace are Base64-encoded + /// so every value stays on a single line. /// class TestListener (Instrumentation instrumentation) : ITestListener { + // Status codes mirror AndroidJUnitRunner conventions so the values are + // familiar, but the host relies on the explicit "event"/"outcome" keys. + const int StatusStart = 1; + const int StatusPassed = 0; + const int StatusFailed = -2; + const int StatusSkipped = -3; + public void TestStarted (ITest test) { if (test.IsSuite) return; Log.Info (LogTag, $"[START] {test.FullName}"); + + var b = new Bundle (); + b.PutString ("event", "start"); + b.PutString ("test", test.FullName); + b.PutString ("name", test.Name); + b.PutString ("class", test.ClassName ?? ""); + instrumentation.SendStatus (StatusStart, b); } public void TestFinished (ITestResult result) @@ -327,20 +350,30 @@ public void TestFinished (ITestResult result) if (result.Test.IsSuite) return; - var outcome = result.ResultState.Status switch { - TestStatus.Passed => "passed", - TestStatus.Failed => "failed", - _ => "skipped", + var (outcome, statusCode) = result.ResultState.Status switch { + TestStatus.Passed => ("passed", StatusPassed), + TestStatus.Failed => ("failed", StatusFailed), + _ => ("skipped", StatusSkipped), }; Log.Info (LogTag, $"[{outcome.ToUpperInvariant ()}] {result.FullName}"); var b = new Bundle (); + b.PutString ("event", "finish"); b.PutString ("test", result.FullName); + b.PutString ("name", result.Test.Name); + b.PutString ("class", result.Test.ClassName ?? ""); b.PutString ("outcome", outcome); - instrumentation.SendStatus (0, b); + if (result.Message is not null) + b.PutString ("message-b64", Encode (result.Message)); + if (result.StackTrace is not null) + b.PutString ("stack-b64", Encode (result.StackTrace)); + instrumentation.SendStatus (statusCode, b); } + static string Encode (string value) + => Convert.ToBase64String (System.Text.Encoding.UTF8.GetBytes (value)); + public void TestOutput (TestOutput output) { } public void SendMessage (TestMessage message) { } } From 538330d252a7b4e0c8a653a169b32c420bd15876 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 11:31:34 +0200 Subject: [PATCH 02/10] Move ServerCertificateCustomValidationCallback [Ignore] out of this PR The [Ignore] for the crashing AndroidMessageHandlerTests .ServerCertificateCustomValidationCallback_* tests (dotnet/android#8608) belongs with the trimmable-typemap work; move it to #11801. This PR is now scoped to the MTP streaming-results change only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Xamarin.Android.Net/AndroidMessageHandlerTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs b/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs index 315bb04bac6..99cde29dcd3 100644 --- a/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs +++ b/tests/Mono.Android-Tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs @@ -137,7 +137,6 @@ public async Task DoesNotDisposeContentStream() } [Test] - [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_ApproveRequest () { bool callbackHasBeenCalled = false; @@ -164,7 +163,6 @@ public async Task ServerCertificateCustomValidationCallback_ApproveRequest () } [Test] - [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_RejectRequest () { bool callbackHasBeenCalled = false; @@ -183,7 +181,6 @@ public async Task ServerCertificateCustomValidationCallback_RejectRequest () } [Test] - [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_ApprovesRequestWithInvalidCertificate () { bool callbackHasBeenCalled = false; @@ -211,7 +208,6 @@ public async Task NoServerCertificateCustomValidationCallback_ThrowsWhenThereIsC } [Test] - [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_IgnoresCertificateHostnameMismatch () { bool callbackHasBeenCalled = false; @@ -233,7 +229,6 @@ public async Task ServerCertificateCustomValidationCallback_IgnoresCertificateHo } [Test] - [Ignore ("Crashes the test process with a native SIGSEGV: https://github.com/dotnet/android/issues/8608")] public async Task ServerCertificateCustomValidationCallback_Redirects () { int callbackCounter = 0; From fc34441adc7bd02d4ee87b0d33419f60ede354ea Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 11:36:50 +0200 Subject: [PATCH 03/10] Only report streamed status blocks with explicit event/outcome Address review feedback: PublishStatusAsync treated any INSTRUMENTATION_STATUS block with a "test" key as a finished test (defaulting a missing outcome to Passed) and added it to reportedFinal. For instrumentation that doesn't use this streaming protocol (e.g. AndroidJUnitRunner), that would mis-report tests as passed and, by making reportedFinal non-empty, suppress the TRX fallback. Now only blocks with event=="start"/"finish" are handled, and a finish must carry a non-empty outcome; anything else is ignored so ReportFromTrxAsync can handle the run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AndroidTestAdapter.cs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index a34c9332e18..d9dc72e7b0c 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -153,14 +153,23 @@ async Task RunInstrumentationOnDeviceAsync (Execute /// /// Publishes a single streamed instrumentation status block to MTP: a "start" /// event becomes an in-progress node; a "finish" event becomes the final - /// pass/fail/skip node. + /// pass/fail/skip node. Blocks that aren't part of this streaming protocol + /// (no recognized "event", or a "finish" without an "outcome") are ignored so + /// they neither mis-report a test nor suppress the TRX fallback. /// async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationStatus status, HashSet reportedFinal) { + // Only handle our explicit streaming protocol. Other instrumentation + // (e.g. AndroidJUnitRunner) emits status blocks with a "test" key but no + // "event"/"outcome"; treating those as results would report them as + // passed and, worse, mark reportedFinal non-empty so the TRX fallback + // never runs. Skip them and let ReportFromTrxAsync handle the run. + if (!status.Values.TryGetValue ("event", out var eventKind) || (eventKind != "start" && eventKind != "finish")) + return; + if (!status.Values.TryGetValue ("test", out var fullyQualifiedName) || string.IsNullOrEmpty (fullyQualifiedName)) return; - status.Values.TryGetValue ("event", out var eventKind); status.Values.TryGetValue ("name", out var displayName); status.Values.TryGetValue ("class", out var className); @@ -174,7 +183,10 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session return; } - status.Values.TryGetValue ("outcome", out var outcome); + // eventKind == "finish": a valid completion must carry an outcome. + if (!status.Values.TryGetValue ("outcome", out var outcome) || string.IsNullOrEmpty (outcome)) + return; + var errorMessage = DecodeOrNull (status.Values, "message-b64"); var stackTrace = DecodeOrNull (status.Values, "stack-b64"); From add9f4f2d027e3dd4eea7a3f4024c7247201f240 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 11:49:23 +0200 Subject: [PATCH 04/10] Harden crash handling and always observe the reporting task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Research finding: MTP's built-in crash handling (ITestHostProcessLifetimeHandler, CrashDump/hang-dump) only covers the *test host* process — i.e. Microsoft.Android.Run itself — not the app-under-test on the device. A device-side native crash is invisible to MTP, so a synthetic report is the only option. MTP does, however, provide ErrorTestNodeStateProperty for infrastructure errors (distinct from an assertion Failed), which is the right state for a crash. Improvements: * Track the in-flight test (the one whose "start" arrived without a "finish"). On a crash, resolve *that* test from InProgress to ErrorTestNodeStateProperty so it isn't left dangling and the crash is attributed to the offending test. Only when no test was in flight (crash between tests) do we publish the synthetic ".ProcessCrashed" error node. Either way the run is clearly marked failed, and we never emit two failures for one crash. * Use ErrorTestNodeStateProperty (not FailedTestNodeStateProperty) for crash and unrecognized-outcome nodes — these are infrastructure errors, not assertion failures. * Error handling: a single unreportable status block is logged and skipped instead of aborting the whole run; OperationCanceledException is rethrown, not swallowed. * Always observe the consumer task: `await consumer` moved into the `finally` so it runs even when StartProcess throws or is cancelled (previously it was skipped on the exception path, leaving the task unobserved). No unit tests are added: Microsoft.Android.Run has no test project, and the adapter is exercised end-to-end by every on-device test run in CI. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AndroidTestAdapter.cs | 114 +++++++++++++----- 1 file changed, 83 insertions(+), 31 deletions(-) diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index d9dc72e7b0c..6d0d475c76b 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -53,33 +53,35 @@ public async Task ExecuteRequestAsync (ExecuteRequestContext context) async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionUid) { - // Track tests we've already reported (by their final outcome) as they - // stream in, so we don't double-report them from the pulled TRX. - var reportedFinal = new HashSet (StringComparer.Ordinal); + // Shared state for the streamed run: which tests we've already reported a + // final outcome for (so we don't double-report them from the pulled TRX), + // and which test — if any — is currently executing (so a crash can resolve + // it to a terminal state instead of leaving it "in progress"). + var state = new StreamingState (); // 1. Run instrumentation on device, publishing each test to MTP live as // its INSTRUMENTATION_STATUS block arrives. - var bundleResults = await RunInstrumentationOnDeviceAsync (context, sessionUid, reportedFinal, context.CancellationToken); + var bundleResults = await RunInstrumentationOnDeviceAsync (context, sessionUid, state, context.CancellationToken); if (verbose) { - Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}, streamed={reportedFinal.Count}"); + Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}, streamed={state.ReportedFinal.Count}"); if (bundleResults.ResultsPath != null) Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); } // 2. If we streamed at least one completed test, streaming is authoritative // (it's resilient to a mid-run crash). Otherwise fall back to the TRX. - if (reportedFinal.Count == 0) { + if (state.ReportedFinal.Count == 0) { await ReportFromTrxAsync (context, sessionUid, bundleResults); return; } // 3. We streamed live results. If the run did not finish cleanly (the app - // process crashed before writing the final results bundle), surface a - // synthetic failed test so the run is clearly marked failed rather than - // silently dropping the in-flight/never-run tests. + // process crashed before writing the final results bundle), surface the + // crash so the run is clearly marked failed rather than silently dropping + // the in-flight/never-run tests. if (bundleResults.Crashed) - await PublishCrashNodeAsync (context, sessionUid, bundleResults); + await PublishCrashAsync (context, sessionUid, bundleResults, state); } /// @@ -87,7 +89,7 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU /// the streamed 'am instrument -w -r' output, and returns the parsed final /// results bundle (summary counts, resultsPath, error, crash state). /// - async Task RunInstrumentationOnDeviceAsync (ExecuteRequestContext context, SessionUid sessionUid, HashSet reportedFinal, CancellationToken cancellationToken) + async Task RunInstrumentationOnDeviceAsync (ExecuteRequestContext context, SessionUid sessionUid, StreamingState state, CancellationToken cancellationToken) { // '-r' prints raw INSTRUMENTATION_STATUS results (so we can stream them); // '-w' waits for the run to complete. @@ -110,22 +112,33 @@ async Task RunInstrumentationOnDeviceAsync (Execute var consumer = Task.Run (async () => { await foreach (var status in channel.Reader.ReadAllAsync ()) { - await PublishStatusAsync (context, sessionUid, status, reportedFinal); + try { + await PublishStatusAsync (context, sessionUid, status, state); + } catch (OperationCanceledException) { + throw; + } catch (Exception ex) { + // One malformed/unreportable status block must not abort + // reporting for the rest of the run. + Console.Error.WriteLine ($"[AndroidTestAdapter] Failed to report a streamed test result: {ex}"); + } } }); var parser = new StatusStreamParser (channel.Writer, fullOutput, verbose); using var stdout = new LineWriter (parser.OnLine); - int exitCode; + int exitCode = 0; try { exitCode = await ProcessUtils.StartProcess (psi, stdout, stderr, cancellationToken); } finally { + // Flush the trailing partial line, discard any unterminated status + // block, then complete the channel and always observe the consumer — + // even if StartProcess threw or was cancelled. stdout.Flush (); parser.Complete (); channel.Writer.Complete (); + await consumer; } - await consumer; var output = fullOutput.ToString (); if (verbose) { @@ -142,7 +155,7 @@ async Task RunInstrumentationOnDeviceAsync (Execute exitCode != 0 || output.Contains ("INSTRUMENTATION_FAILED", StringComparison.Ordinal) || output.Contains ("Process crashed", StringComparison.Ordinal) || - (reportedFinal.Count > 0 && result.ResultsPath == null); + (state.ReportedFinal.Count > 0 && result.ResultsPath == null); if (result.Crashed && result.Error == null) result.Error = ExtractCrashMessage (output) ?? (exitCode != 0 ? $"Instrumentation exited with code {exitCode}." : "The test process terminated before reporting a result (likely a native crash)."); @@ -157,12 +170,12 @@ async Task RunInstrumentationOnDeviceAsync (Execute /// (no recognized "event", or a "finish" without an "outcome") are ignored so /// they neither mis-report a test nor suppress the TRX fallback. /// - async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationStatus status, HashSet reportedFinal) + async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationStatus status, StreamingState state) { // Only handle our explicit streaming protocol. Other instrumentation // (e.g. AndroidJUnitRunner) emits status blocks with a "test" key but no // "event"/"outcome"; treating those as results would report them as - // passed and, worse, mark reportedFinal non-empty so the TRX fallback + // passed and, worse, mark ReportedFinal non-empty so the TRX fallback // never runs. Skip them and let ReportFromTrxAsync handle the run. if (!status.Values.TryGetValue ("event", out var eventKind) || (eventKind != "start" && eventKind != "finish")) return; @@ -174,6 +187,10 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session status.Values.TryGetValue ("class", out var className); if (eventKind == "start") { + // Remember the running test so a crash can resolve it to a terminal + // state instead of leaving a dangling "in progress" node. + state.InFlightUid = fullyQualifiedName; + state.InFlightName = displayName; var startNode = new TestNode { Uid = new TestNodeUid (fullyQualifiedName), DisplayName = displayName ?? fullyQualifiedName, @@ -187,6 +204,12 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session if (!status.Values.TryGetValue ("outcome", out var outcome) || string.IsNullOrEmpty (outcome)) return; + // The test completed, so it's no longer in flight. + if (state.InFlightUid == fullyQualifiedName) { + state.InFlightUid = null; + state.InFlightName = null; + } + var errorMessage = DecodeOrNull (status.Values, "message-b64"); var stackTrace = DecodeOrNull (status.Values, "stack-b64"); @@ -194,11 +217,13 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session if (!string.IsNullOrEmpty (stackTrace)) failureMessage += "\n" + stackTrace; + // An unrecognized outcome is reported as an error rather than silently + // counted as a pass. var stateProperty = outcome switch { "passed" => (IProperty) new PassedTestNodeStateProperty (), "failed" => new FailedTestNodeStateProperty (failureMessage), "skipped" => new SkippedTestNodeStateProperty (errorMessage), - _ => new PassedTestNodeStateProperty (), + _ => new ErrorTestNodeStateProperty ($"Unrecognized test outcome '{outcome}'."), }; var properties = new List { stateProperty }; @@ -214,27 +239,43 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session }; await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, testNode)); - reportedFinal.Add (fullyQualifiedName); + state.ReportedFinal.Add (fullyQualifiedName); } /// - /// Publishes a synthetic failed test node representing a mid-run process crash, - /// so the overall run is reported as failed and the crash is visible in results. + /// Reports a mid-run process crash. If a test was executing when the process + /// died, that test is resolved from "in progress" to an + /// (an infrastructure error, not an assertion failure) so it isn't left + /// dangling and the crash is attributed to it. Otherwise a synthetic error + /// node is published so the overall run is still clearly marked failed. /// - async Task PublishCrashNodeAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults) + async Task PublishCrashAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults, StreamingState state) { var message = bundleResults.Error ?? "The test process terminated before all tests completed (likely a native crash)."; Console.Error.WriteLine ($"[AndroidTestAdapter] {message}"); - var crashUid = $"{instrumentation}.ProcessCrashed"; - var crashNode = new TestNode { - Uid = new TestNodeUid (crashUid), - DisplayName = "Test process crashed before completion", - Properties = new PropertyBag ( - new FailedTestNodeStateProperty (message), - new TrxFullyQualifiedTypeNameProperty (instrumentation), - new TrxExceptionProperty (message, null)), - }; + TestNode crashNode; + if (state.InFlightUid != null) { + crashNode = new TestNode { + Uid = new TestNodeUid (state.InFlightUid), + DisplayName = state.InFlightName ?? state.InFlightUid, + Properties = new PropertyBag ( + new ErrorTestNodeStateProperty ($"The test process crashed while running this test.\n{message}"), + new TrxExceptionProperty (message, null)), + }; + state.ReportedFinal.Add (state.InFlightUid); + state.InFlightUid = null; + state.InFlightName = null; + } else { + crashNode = new TestNode { + Uid = new TestNodeUid ($"{instrumentation}.ProcessCrashed"), + DisplayName = "Test process crashed before completion", + Properties = new PropertyBag ( + new ErrorTestNodeStateProperty (message), + new TrxFullyQualifiedTypeNameProperty (instrumentation), + new TrxExceptionProperty (message, null)), + }; + } await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, crashNode)); } @@ -477,6 +518,17 @@ class InstrumentationBundleResult public bool Crashed { get; set; } } +/// +/// Mutable state shared across a streamed run: the tests already reported with a +/// final outcome, and the test currently executing (if any). +/// +sealed class StreamingState +{ + public HashSet ReportedFinal { get; } = new (StringComparer.Ordinal); + public string? InFlightUid { get; set; } + public string? InFlightName { get; set; } +} + /// /// A single completed INSTRUMENTATION_STATUS block: its key/value pairs /// plus the trailing INSTRUMENTATION_STATUS_CODE. From 3e7d39c1229955231db91c9f18183bea579d4c6d Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 13:22:30 +0200 Subject: [PATCH 05/10] Cast instrumentation status codes to Android.App.Result Instrumentation.SendStatus's resultCode parameter is enumified to Android.App.Result (like Finish), so the raw int status codes could not be passed implicitly. Cast explicitly to preserve the intended instrumentation report codes (1/0/-2/-3). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/TestRunner.Core/TestInstrumentation.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/TestRunner.Core/TestInstrumentation.cs b/tests/TestRunner.Core/TestInstrumentation.cs index 5f849f08ee7..d592c5caec8 100644 --- a/tests/TestRunner.Core/TestInstrumentation.cs +++ b/tests/TestRunner.Core/TestInstrumentation.cs @@ -342,7 +342,7 @@ public void TestStarted (ITest test) b.PutString ("test", test.FullName); b.PutString ("name", test.Name); b.PutString ("class", test.ClassName ?? ""); - instrumentation.SendStatus (StatusStart, b); + instrumentation.SendStatus ((Result) StatusStart, b); } public void TestFinished (ITestResult result) @@ -368,7 +368,7 @@ public void TestFinished (ITestResult result) b.PutString ("message-b64", Encode (result.Message)); if (result.StackTrace is not null) b.PutString ("stack-b64", Encode (result.StackTrace)); - instrumentation.SendStatus (statusCode, b); + instrumentation.SendStatus ((Result) statusCode, b); } static string Encode (string value) From c25c09ee0b710adf531b7fe386b73c575270367e Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 13:22:30 +0200 Subject: [PATCH 06/10] Address review: fix ReportedFinal comments and stray blank line - Update the ReportedFinal comments to describe its real role (a streamed-anything / authoritative-streaming signal) instead of TRX dedup, which never actually runs. - Remove a double blank line before enum TrxOutcome. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Microsoft.Android.Run/AndroidTestAdapter.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index 6d0d475c76b..ef367cbac04 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -53,10 +53,11 @@ public async Task ExecuteRequestAsync (ExecuteRequestContext context) async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionUid) { - // Shared state for the streamed run: which tests we've already reported a - // final outcome for (so we don't double-report them from the pulled TRX), - // and which test — if any — is currently executing (so a crash can resolve - // it to a terminal state instead of leaving it "in progress"). + // Shared state for the streamed run: the set of tests we've streamed a + // final outcome for (a non-empty set means streaming was authoritative, so + // we skip the TRX fallback), and which test — if any — is currently + // executing (so a crash can resolve it to a terminal state instead of + // leaving it "in progress"). var state = new StreamingState (); // 1. Run instrumentation on device, publishing each test to MTP live as @@ -519,8 +520,9 @@ class InstrumentationBundleResult } /// -/// Mutable state shared across a streamed run: the tests already reported with a -/// final outcome, and the test currently executing (if any). +/// Mutable state shared across a streamed run: the tests streamed with a final +/// outcome (a non-empty set means streaming was authoritative, so the TRX +/// fallback is skipped), and the test currently executing (if any). /// sealed class StreamingState { @@ -640,7 +642,6 @@ public void Complete () } } - enum TrxOutcome { Passed, From 282a755407b755487a50e4b280f8b09bfcc1d926 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 1 Jul 2026 14:09:20 +0200 Subject: [PATCH 07/10] Stream per-test progress to the CI log Echo every finished test (including passes) to the console as it streams in, with a running count, so a long device run's log grows steadily and progress stays visible instead of appearing to hang. MTP captures the test host's console output by default, so this stays silent for local runs. CI opts in via -p:TestingPlatformCaptureOutput=false on the dotnet test invocation, which lets the lines stream live into the build log. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/yaml-templates/apk-instrumentation.yaml | 2 +- src/Microsoft.Android.Run/AndroidTestAdapter.cs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml index a06d07625d4..ecd80063c19 100644 --- a/build-tools/automation/yaml-templates/apk-instrumentation.yaml +++ b/build-tools/automation/yaml-templates/apk-instrumentation.yaml @@ -44,7 +44,7 @@ steps: $env:PATH = "${DOTNET_ROOT};$env:PATH" $dotnetPath = "${DOTNET_ROOT}\dotnet.exe" } - & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx ${{ parameters.extraBuildArgs }} + & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx -p:TestingPlatformCaptureOutput=false ${{ parameters.extraBuildArgs }} Pop-Location displayName: run ${{ parameters.testName }} condition: ${{ parameters.condition }} diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index ef367cbac04..87afb330972 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -241,6 +241,14 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, testNode)); state.ReportedFinal.Add (fullyQualifiedName); + + // Echo every finished test — including passes — to the console so the log + // grows steadily and progress stays visible while monitoring a long device + // run (MTP's default output only surfaces failures). MTP captures the test + // host's console output by default, so this stays silent locally; CI opts + // in by passing -p:TestingPlatformCaptureOutput=false, which lets these + // lines stream live into the build log. + Console.WriteLine ($"[{outcome.ToUpperInvariant ()}] ({state.ReportedFinal.Count}) {fullyQualifiedName}"); } /// From 97ddfeb44a253655ae591a2233b9fc4a56a2aa05 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 2 Jul 2026 10:30:16 +0200 Subject: [PATCH 08/10] [tests] Print passing tests via dotnet test --output Detailed The MTP terminal reporter only prints failed/skipped tests by default. TestingPlatformCaptureOutput=false could not surface passing tests because Microsoft.Android.Run runs in --server dotnettestcli mode, so the terminal is rendered by the dotnet test orchestrator and the child test host's stdout (the [PASSED] echo) is captured and dropped. Use the new dotnet test experience's first-class --output Detailed option to render every outcome (passed/skipped/failed). Passing tests are already published via PassedTestNodeStateProperty, so no adapter change is needed beyond removing the now-redundant [PASSED] console echo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/yaml-templates/apk-instrumentation.yaml | 2 +- src/Microsoft.Android.Run/AndroidTestAdapter.cs | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml index 2084021e850..d3c6bf54183 100644 --- a/build-tools/automation/yaml-templates/apk-instrumentation.yaml +++ b/build-tools/automation/yaml-templates/apk-instrumentation.yaml @@ -74,7 +74,7 @@ steps: $env:PATH = "${DOTNET_ROOT};$env:PATH" $dotnetPath = "${DOTNET_ROOT}\dotnet.exe" } - & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx -p:TestingPlatformCaptureOutput=false ${{ parameters.extraBuildArgs }} + & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx --output Detailed ${{ parameters.extraBuildArgs }} Pop-Location displayName: run ${{ parameters.testName }} condition: ${{ parameters.condition }} diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index 87afb330972..ef367cbac04 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -241,14 +241,6 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, testNode)); state.ReportedFinal.Add (fullyQualifiedName); - - // Echo every finished test — including passes — to the console so the log - // grows steadily and progress stays visible while monitoring a long device - // run (MTP's default output only surfaces failures). MTP captures the test - // host's console output by default, so this stays silent locally; CI opts - // in by passing -p:TestingPlatformCaptureOutput=false, which lets these - // lines stream live into the build log. - Console.WriteLine ($"[{outcome.ToUpperInvariant ()}] ({state.ReportedFinal.Count}) {fullyQualifiedName}"); } /// From dc779a345b65be7089323680e0b9a5262eb1cf03 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 2 Jul 2026 10:49:28 +0200 Subject: [PATCH 09/10] [tests] Address review: shared protocol constants + status-code enum Replace the magic instrumentation-protocol strings ("event"/"finish"/ "passed"/... and the bundle keys) that were duplicated across the device test runner and the host adapter with a single source-linked contract, InstrumentationProtocol, so the two sides cannot drift. Convert the device-side am instrument status codes to a named InstrumentationStatusCode enum. These are Android instrumentation Result codes (mirroring AndroidJUnitRunner 1/0/-2/-3), not MTP results, so no Microsoft.Testing.* enum applies; the host keys off the explicit event/outcome values rather than the numeric code. Addresses code review feedback on PR #11833. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AndroidTestAdapter.cs | 34 +++++----- .../InstrumentationProtocol.cs | 36 +++++++++++ tests/TestRunner.Core/TestInstrumentation.cs | 64 +++++++++++-------- .../TestRunner.Core.NET.csproj | 6 ++ 4 files changed, 95 insertions(+), 45 deletions(-) create mode 100644 src/Microsoft.Android.Run/InstrumentationProtocol.cs diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index ef367cbac04..ae5d68329cc 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -178,16 +178,16 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session // "event"/"outcome"; treating those as results would report them as // passed and, worse, mark ReportedFinal non-empty so the TRX fallback // never runs. Skip them and let ReportFromTrxAsync handle the run. - if (!status.Values.TryGetValue ("event", out var eventKind) || (eventKind != "start" && eventKind != "finish")) + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyEvent, out var eventKind) || (eventKind != InstrumentationProtocol.EventStart && eventKind != InstrumentationProtocol.EventFinish)) return; - if (!status.Values.TryGetValue ("test", out var fullyQualifiedName) || string.IsNullOrEmpty (fullyQualifiedName)) + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyTest, out var fullyQualifiedName) || string.IsNullOrEmpty (fullyQualifiedName)) return; - status.Values.TryGetValue ("name", out var displayName); - status.Values.TryGetValue ("class", out var className); + status.Values.TryGetValue (InstrumentationProtocol.KeyName, out var displayName); + status.Values.TryGetValue (InstrumentationProtocol.KeyClass, out var className); - if (eventKind == "start") { + if (eventKind == InstrumentationProtocol.EventStart) { // Remember the running test so a crash can resolve it to a terminal // state instead of leaving a dangling "in progress" node. state.InFlightUid = fullyQualifiedName; @@ -202,7 +202,7 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session } // eventKind == "finish": a valid completion must carry an outcome. - if (!status.Values.TryGetValue ("outcome", out var outcome) || string.IsNullOrEmpty (outcome)) + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyOutcome, out var outcome) || string.IsNullOrEmpty (outcome)) return; // The test completed, so it's no longer in flight. @@ -211,8 +211,8 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session state.InFlightName = null; } - var errorMessage = DecodeOrNull (status.Values, "message-b64"); - var stackTrace = DecodeOrNull (status.Values, "stack-b64"); + var errorMessage = DecodeOrNull (status.Values, InstrumentationProtocol.KeyMessageBase64); + var stackTrace = DecodeOrNull (status.Values, InstrumentationProtocol.KeyStackBase64); var failureMessage = errorMessage ?? "Test failed"; if (!string.IsNullOrEmpty (stackTrace)) @@ -221,16 +221,16 @@ async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid session // An unrecognized outcome is reported as an error rather than silently // counted as a pass. var stateProperty = outcome switch { - "passed" => (IProperty) new PassedTestNodeStateProperty (), - "failed" => new FailedTestNodeStateProperty (failureMessage), - "skipped" => new SkippedTestNodeStateProperty (errorMessage), + InstrumentationProtocol.OutcomePassed => (IProperty) new PassedTestNodeStateProperty (), + InstrumentationProtocol.OutcomeFailed => new FailedTestNodeStateProperty (failureMessage), + InstrumentationProtocol.OutcomeSkipped => new SkippedTestNodeStateProperty (errorMessage), _ => new ErrorTestNodeStateProperty ($"Unrecognized test outcome '{outcome}'."), }; var properties = new List { stateProperty }; if (!string.IsNullOrEmpty (className)) properties.Add (new TrxFullyQualifiedTypeNameProperty (className)); - if (outcome == "failed" && (!string.IsNullOrEmpty (errorMessage) || !string.IsNullOrEmpty (stackTrace))) + if (outcome == InstrumentationProtocol.OutcomeFailed && (!string.IsNullOrEmpty (errorMessage) || !string.IsNullOrEmpty (stackTrace))) properties.Add (new TrxExceptionProperty (errorMessage, stackTrace)); var testNode = new TestNode { @@ -425,15 +425,15 @@ static InstrumentationBundleResult ParseInstrumentationBundle (string output, st } } - if (bundleValues.TryGetValue ("passed", out var passedStr) && int.TryParse (passedStr, out int passed)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyPassedCount, out var passedStr) && int.TryParse (passedStr, out int passed)) result.Passed = passed; - if (bundleValues.TryGetValue ("failed", out var failedStr) && int.TryParse (failedStr, out int failed)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyFailedCount, out var failedStr) && int.TryParse (failedStr, out int failed)) result.Failed = failed; - if (bundleValues.TryGetValue ("skipped", out var skippedStr) && int.TryParse (skippedStr, out int skipped)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeySkippedCount, out var skippedStr) && int.TryParse (skippedStr, out int skipped)) result.Skipped = skipped; - if (bundleValues.TryGetValue ("resultsPath", out var resultsPath)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyResultsPath, out var resultsPath)) result.ResultsPath = resultsPath; - if (bundleValues.TryGetValue ("error", out var bundleError)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyError, out var bundleError)) result.Error = bundleError; // Surface adb stderr if no results were parsed diff --git a/src/Microsoft.Android.Run/InstrumentationProtocol.cs b/src/Microsoft.Android.Run/InstrumentationProtocol.cs new file mode 100644 index 00000000000..52d85679470 --- /dev/null +++ b/src/Microsoft.Android.Run/InstrumentationProtocol.cs @@ -0,0 +1,36 @@ +/// +/// Wire contract for the on-device test instrumentation streaming protocol. The +/// device side (Xamarin.Android.UnitTests.TestInstrumentation) emits +/// INSTRUMENTATION_STATUS/INSTRUMENTATION_RESULT bundles through +/// am instrument -r, and the host side (AndroidTestAdapter) parses +/// them. Values are kept as human-readable strings so the raw instrumentation +/// output stays legible in logcat. This file is source-linked into both the +/// device test runner and the host so the two sides cannot drift out of sync. +/// +static class InstrumentationProtocol +{ + // Keys used in the per-test streaming status blocks. + public const string KeyEvent = "event"; + public const string KeyTest = "test"; + public const string KeyName = "name"; + public const string KeyClass = "class"; + public const string KeyOutcome = "outcome"; + public const string KeyMessageBase64 = "message-b64"; + public const string KeyStackBase64 = "stack-b64"; + + // Keys used in the final results bundle. + public const string KeyPassedCount = "passed"; + public const string KeyFailedCount = "failed"; + public const string KeySkippedCount = "skipped"; + public const string KeyResultsPath = "resultsPath"; + public const string KeyError = "error"; + + // "event" values. + public const string EventStart = "start"; + public const string EventFinish = "finish"; + + // "outcome" values. + public const string OutcomePassed = "passed"; + public const string OutcomeFailed = "failed"; + public const string OutcomeSkipped = "skipped"; +} diff --git a/tests/TestRunner.Core/TestInstrumentation.cs b/tests/TestRunner.Core/TestInstrumentation.cs index d592c5caec8..1cbd327ae98 100644 --- a/tests/TestRunner.Core/TestInstrumentation.cs +++ b/tests/TestRunner.Core/TestInstrumentation.cs @@ -106,14 +106,14 @@ public override void OnStart () Log.Info (LogTag, $"TRX written to: {trxPath}"); Log.Info (LogTag, $"Results: passed={passed}, failed={failed}, skipped={skipped}"); - bundle.PutInt ("passed", passed); - bundle.PutInt ("failed", failed); - bundle.PutInt ("skipped", skipped); - bundle.PutString ("resultsPath", trxPath); + bundle.PutInt (InstrumentationProtocol.KeyPassedCount, passed); + bundle.PutInt (InstrumentationProtocol.KeyFailedCount, failed); + bundle.PutInt (InstrumentationProtocol.KeySkippedCount, skipped); + bundle.PutString (InstrumentationProtocol.KeyResultsPath, trxPath); Finish (Result.Ok, bundle); } catch (Exception ex) { Log.Error (LogTag, $"Test run failed: {ex}"); - bundle.PutString ("error", ex.ToString ()); + bundle.PutString (InstrumentationProtocol.KeyError, ex.ToString ()); Finish (Result.Canceled, bundle); } } @@ -322,15 +322,23 @@ static void WriteTrxFile (string path, List assemblyResults) /// (potentially multi-line) failure message and stack trace are Base64-encoded /// so every value stays on a single line. /// - class TestListener (Instrumentation instrumentation) : ITestListener + /// + /// Android am instrument status codes emitted on the + /// INSTRUMENTATION_STATUS_CODE line. The values mirror + /// AndroidJUnitRunner's conventions so tools scraping the raw output see + /// familiar codes; the host keys off the explicit event/outcome + /// bundle values rather than these numbers. + /// + enum InstrumentationStatusCode { - // Status codes mirror AndroidJUnitRunner conventions so the values are - // familiar, but the host relies on the explicit "event"/"outcome" keys. - const int StatusStart = 1; - const int StatusPassed = 0; - const int StatusFailed = -2; - const int StatusSkipped = -3; + Passed = 0, + Start = 1, + Failed = -2, + Skipped = -3, + } + class TestListener (Instrumentation instrumentation) : ITestListener + { public void TestStarted (ITest test) { if (test.IsSuite) @@ -338,11 +346,11 @@ public void TestStarted (ITest test) Log.Info (LogTag, $"[START] {test.FullName}"); var b = new Bundle (); - b.PutString ("event", "start"); - b.PutString ("test", test.FullName); - b.PutString ("name", test.Name); - b.PutString ("class", test.ClassName ?? ""); - instrumentation.SendStatus ((Result) StatusStart, b); + b.PutString (InstrumentationProtocol.KeyEvent, InstrumentationProtocol.EventStart); + b.PutString (InstrumentationProtocol.KeyTest, test.FullName); + b.PutString (InstrumentationProtocol.KeyName, test.Name); + b.PutString (InstrumentationProtocol.KeyClass, test.ClassName ?? ""); + instrumentation.SendStatus ((Result) (int) InstrumentationStatusCode.Start, b); } public void TestFinished (ITestResult result) @@ -351,24 +359,24 @@ public void TestFinished (ITestResult result) return; var (outcome, statusCode) = result.ResultState.Status switch { - TestStatus.Passed => ("passed", StatusPassed), - TestStatus.Failed => ("failed", StatusFailed), - _ => ("skipped", StatusSkipped), + TestStatus.Passed => (InstrumentationProtocol.OutcomePassed, InstrumentationStatusCode.Passed), + TestStatus.Failed => (InstrumentationProtocol.OutcomeFailed, InstrumentationStatusCode.Failed), + _ => (InstrumentationProtocol.OutcomeSkipped, InstrumentationStatusCode.Skipped), }; Log.Info (LogTag, $"[{outcome.ToUpperInvariant ()}] {result.FullName}"); var b = new Bundle (); - b.PutString ("event", "finish"); - b.PutString ("test", result.FullName); - b.PutString ("name", result.Test.Name); - b.PutString ("class", result.Test.ClassName ?? ""); - b.PutString ("outcome", outcome); + b.PutString (InstrumentationProtocol.KeyEvent, InstrumentationProtocol.EventFinish); + b.PutString (InstrumentationProtocol.KeyTest, result.FullName); + b.PutString (InstrumentationProtocol.KeyName, result.Test.Name); + b.PutString (InstrumentationProtocol.KeyClass, result.Test.ClassName ?? ""); + b.PutString (InstrumentationProtocol.KeyOutcome, outcome); if (result.Message is not null) - b.PutString ("message-b64", Encode (result.Message)); + b.PutString (InstrumentationProtocol.KeyMessageBase64, Encode (result.Message)); if (result.StackTrace is not null) - b.PutString ("stack-b64", Encode (result.StackTrace)); - instrumentation.SendStatus ((Result) statusCode, b); + b.PutString (InstrumentationProtocol.KeyStackBase64, Encode (result.StackTrace)); + instrumentation.SendStatus ((Result) (int) statusCode, b); } static string Encode (string value) diff --git a/tests/TestRunner.Core/TestRunner.Core.NET.csproj b/tests/TestRunner.Core/TestRunner.Core.NET.csproj index a2090bd5d1b..269fb5a759c 100644 --- a/tests/TestRunner.Core/TestRunner.Core.NET.csproj +++ b/tests/TestRunner.Core/TestRunner.Core.NET.csproj @@ -22,4 +22,10 @@ + + + + + From f3bc8d8b322f028e02d28b8ef8496b017b4c8c6b Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 2 Jul 2026 17:43:19 +0200 Subject: [PATCH 10/10] [tests] Address review: crash-before-stream + doc-comment placement - Handle a process crash before falling back to the TRX path: when the app dies before any test streams a finish (or after only a start), we now publish the crash node (resolving any in-flight test) instead of calling ReportFromTrxAsync, which has no results file to read and would throw. - Reorder the InstrumentationStatusCode enum above TestListener's doc comment so the streaming documents the class again. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AndroidTestAdapter.cs | 22 +++++++++-------- tests/TestRunner.Core/TestInstrumentation.cs | 24 +++++++++---------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index ae5d68329cc..8abe16c0372 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -70,19 +70,21 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); } - // 2. If we streamed at least one completed test, streaming is authoritative - // (it's resilient to a mid-run crash). Otherwise fall back to the TRX. - if (state.ReportedFinal.Count == 0) { - await ReportFromTrxAsync (context, sessionUid, bundleResults); + // 2. If the run did not finish cleanly (the app process crashed before + // writing the final results bundle), surface the crash. This resolves an + // in-flight streamed test to a terminal state, or publishes a synthetic + // crash node when nothing streamed, instead of falling through to the TRX + // path — which has no results file to read after a crash and would throw. + if (bundleResults.Crashed) { + await PublishCrashAsync (context, sessionUid, bundleResults, state); return; } - // 3. We streamed live results. If the run did not finish cleanly (the app - // process crashed before writing the final results bundle), surface the - // crash so the run is clearly marked failed rather than silently dropping - // the in-flight/never-run tests. - if (bundleResults.Crashed) - await PublishCrashAsync (context, sessionUid, bundleResults, state); + // 3. No crash: if we streamed at least one completed test, streaming is + // authoritative and we're done. Otherwise fall back to the TRX (e.g. an + // older on-device instrumentation that didn't stream results). + if (state.ReportedFinal.Count == 0) + await ReportFromTrxAsync (context, sessionUid, bundleResults); } /// diff --git a/tests/TestRunner.Core/TestInstrumentation.cs b/tests/TestRunner.Core/TestInstrumentation.cs index 1cbd327ae98..afe20802de3 100644 --- a/tests/TestRunner.Core/TestInstrumentation.cs +++ b/tests/TestRunner.Core/TestInstrumentation.cs @@ -310,18 +310,6 @@ static void WriteTrxFile (string path, List assemblyResults) doc.Save (path); } - /// - /// Streams per-test status updates through the instrumentation protocol - /// (am instrument -r) so the host adapter can report each test to MTP - /// as it finishes. This makes results resilient to a mid-run process crash: - /// every test completed before the crash has already been reported, instead - /// of the whole run being lost because the final TRX was never written. - /// - /// Each SendStatus emits an INSTRUMENTATION_STATUS block that - /// the host parses line-by-line. Because that protocol is line-based, the - /// (potentially multi-line) failure message and stack trace are Base64-encoded - /// so every value stays on a single line. - /// /// /// Android am instrument status codes emitted on the /// INSTRUMENTATION_STATUS_CODE line. The values mirror @@ -337,6 +325,18 @@ enum InstrumentationStatusCode Skipped = -3, } + /// + /// Streams per-test status updates through the instrumentation protocol + /// (am instrument -r) so the host adapter can report each test to MTP + /// as it finishes. This makes results resilient to a mid-run process crash: + /// every test completed before the crash has already been reported, instead + /// of the whole run being lost because the final TRX was never written. + /// + /// Each SendStatus emits an INSTRUMENTATION_STATUS block that + /// the host parses line-by-line. Because that protocol is line-based, the + /// (potentially multi-line) failure message and stack trace are Base64-encoded + /// so every value stays on a single line. + /// class TestListener (Instrumentation instrumentation) : ITestListener { public void TestStarted (ITest test)