diff --git a/build-tools/automation/yaml-templates/apk-instrumentation.yaml b/build-tools/automation/yaml-templates/apk-instrumentation.yaml index dd41320d97e..d3c6bf54183 100644 --- a/build-tools/automation/yaml-templates/apk-instrumentation.yaml +++ b/build-tools/automation/yaml-templates/apk-instrumentation.yaml @@ -74,7 +74,7 @@ steps: $env:PATH = "${DOTNET_ROOT};$env:PATH" $dotnetPath = "${DOTNET_ROOT}\dotnet.exe" } - & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx ${{ parameters.extraBuildArgs }} + & $dotnetPath test $projectFile --no-build -bl:${{ parameters.xaSourcePath }}/bin/Test${{ parameters.configuration }}/run-${{ parameters.testName }}.binlog -c ${{ parameters.configuration }} --results-directory $resultsDir --report-trx --report-trx-filename ${{ parameters.testName }}.trx --output Detailed ${{ parameters.extraBuildArgs }} Pop-Location displayName: run ${{ parameters.testName }} condition: ${{ parameters.condition }} diff --git a/src/Microsoft.Android.Run/AndroidTestAdapter.cs b/src/Microsoft.Android.Run/AndroidTestAdapter.cs index bd63505f0ab..8abe16c0372 100644 --- a/src/Microsoft.Android.Run/AndroidTestAdapter.cs +++ b/src/Microsoft.Android.Run/AndroidTestAdapter.cs @@ -1,3 +1,5 @@ +using System.Text; +using System.Threading.Channels; using System.Xml.Linq; using Microsoft.Testing.Extensions.TrxReport.Abstractions; using Microsoft.Testing.Platform.Capabilities.TestFramework; @@ -7,6 +9,7 @@ using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.Requests; using Microsoft.Testing.Platform.TestHost; +using Xamarin.Android.Tools; /// /// A Microsoft Testing Platform test framework adapter that runs Android instrumentation @@ -50,9 +53,242 @@ public async Task ExecuteRequestAsync (ExecuteRequestContext context) async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionUid) { - // 1. Run instrumentation on device - var bundleResults = await RunInstrumentationOnDeviceAsync (context.CancellationToken); + // Shared state for the streamed run: the set of tests we've streamed a + // final outcome for (a non-empty set means streaming was authoritative, so + // we skip the TRX fallback), and which test — if any — is currently + // executing (so a crash can resolve it to a terminal state instead of + // leaving it "in progress"). + var state = new StreamingState (); + // 1. Run instrumentation on device, publishing each test to MTP live as + // its INSTRUMENTATION_STATUS block arrives. + var bundleResults = await RunInstrumentationOnDeviceAsync (context, sessionUid, state, context.CancellationToken); + + if (verbose) { + Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}, streamed={state.ReportedFinal.Count}"); + if (bundleResults.ResultsPath != null) + Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); + } + + // 2. If the run did not finish cleanly (the app process crashed before + // writing the final results bundle), surface the crash. This resolves an + // in-flight streamed test to a terminal state, or publishes a synthetic + // crash node when nothing streamed, instead of falling through to the TRX + // path — which has no results file to read after a crash and would throw. + if (bundleResults.Crashed) { + await PublishCrashAsync (context, sessionUid, bundleResults, state); + return; + } + + // 3. No crash: if we streamed at least one completed test, streaming is + // authoritative and we're done. Otherwise fall back to the TRX (e.g. an + // older on-device instrumentation that didn't stream results). + if (state.ReportedFinal.Count == 0) + await ReportFromTrxAsync (context, sessionUid, bundleResults); + } + + /// + /// Publishes each test to MTP as its INSTRUMENTATION_STATUS block arrives from + /// the streamed 'am instrument -w -r' output, and returns the parsed final + /// results bundle (summary counts, resultsPath, error, crash state). + /// + async Task RunInstrumentationOnDeviceAsync (ExecuteRequestContext context, SessionUid sessionUid, StreamingState state, CancellationToken cancellationToken) + { + // '-r' prints raw INSTRUMENTATION_STATUS results (so we can stream them); + // '-w' waits for the run to complete. + var cmdArgs = $"shell am instrument -w -r {package}/{instrumentation}"; + var psi = AdbHelper.CreateStartInfo (adbPath, adbTarget, cmdArgs); + + if (verbose) + Console.WriteLine ($"Running: adb {psi.Arguments}"); + + // Completed status blocks are handed off to a single async consumer that + // publishes them to MTP, so the synchronous stdout read loop is never + // blocked on an async PublishAsync. + var channel = Channel.CreateUnbounded (new UnboundedChannelOptions { + SingleReader = true, + SingleWriter = true, + }); + + var fullOutput = new StringBuilder (); + using var stderr = new StringWriter (); + + var consumer = Task.Run (async () => { + await foreach (var status in channel.Reader.ReadAllAsync ()) { + try { + await PublishStatusAsync (context, sessionUid, status, state); + } catch (OperationCanceledException) { + throw; + } catch (Exception ex) { + // One malformed/unreportable status block must not abort + // reporting for the rest of the run. + Console.Error.WriteLine ($"[AndroidTestAdapter] Failed to report a streamed test result: {ex}"); + } + } + }); + + var parser = new StatusStreamParser (channel.Writer, fullOutput, verbose); + using var stdout = new LineWriter (parser.OnLine); + + int exitCode = 0; + try { + exitCode = await ProcessUtils.StartProcess (psi, stdout, stderr, cancellationToken); + } finally { + // Flush the trailing partial line, discard any unterminated status + // block, then complete the channel and always observe the consumer — + // even if StartProcess threw or was cancelled. + stdout.Flush (); + parser.Complete (); + channel.Writer.Complete (); + await consumer; + } + + var output = fullOutput.ToString (); + if (verbose) { + Console.WriteLine ($"[AndroidTestAdapter] Exit code: {exitCode}"); + Console.WriteLine (output); + } + + var result = ParseInstrumentationBundle (output, stderr.ToString ()); + + // The final bundle (with resultsPath) is only emitted if Finish() ran to + // completion. Its absence — or a non-zero exit / an explicit crash marker — + // means the app process died mid-run. + result.Crashed = + exitCode != 0 || + output.Contains ("INSTRUMENTATION_FAILED", StringComparison.Ordinal) || + output.Contains ("Process crashed", StringComparison.Ordinal) || + (state.ReportedFinal.Count > 0 && result.ResultsPath == null); + + if (result.Crashed && result.Error == null) + result.Error = ExtractCrashMessage (output) ?? (exitCode != 0 ? $"Instrumentation exited with code {exitCode}." : "The test process terminated before reporting a result (likely a native crash)."); + + return result; + } + + /// + /// Publishes a single streamed instrumentation status block to MTP: a "start" + /// event becomes an in-progress node; a "finish" event becomes the final + /// pass/fail/skip node. Blocks that aren't part of this streaming protocol + /// (no recognized "event", or a "finish" without an "outcome") are ignored so + /// they neither mis-report a test nor suppress the TRX fallback. + /// + async Task PublishStatusAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationStatus status, StreamingState state) + { + // Only handle our explicit streaming protocol. Other instrumentation + // (e.g. AndroidJUnitRunner) emits status blocks with a "test" key but no + // "event"/"outcome"; treating those as results would report them as + // passed and, worse, mark ReportedFinal non-empty so the TRX fallback + // never runs. Skip them and let ReportFromTrxAsync handle the run. + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyEvent, out var eventKind) || (eventKind != InstrumentationProtocol.EventStart && eventKind != InstrumentationProtocol.EventFinish)) + return; + + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyTest, out var fullyQualifiedName) || string.IsNullOrEmpty (fullyQualifiedName)) + return; + + status.Values.TryGetValue (InstrumentationProtocol.KeyName, out var displayName); + status.Values.TryGetValue (InstrumentationProtocol.KeyClass, out var className); + + if (eventKind == InstrumentationProtocol.EventStart) { + // Remember the running test so a crash can resolve it to a terminal + // state instead of leaving a dangling "in progress" node. + state.InFlightUid = fullyQualifiedName; + state.InFlightName = displayName; + var startNode = new TestNode { + Uid = new TestNodeUid (fullyQualifiedName), + DisplayName = displayName ?? fullyQualifiedName, + Properties = new PropertyBag (new InProgressTestNodeStateProperty ()), + }; + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, startNode)); + return; + } + + // eventKind == "finish": a valid completion must carry an outcome. + if (!status.Values.TryGetValue (InstrumentationProtocol.KeyOutcome, out var outcome) || string.IsNullOrEmpty (outcome)) + return; + + // The test completed, so it's no longer in flight. + if (state.InFlightUid == fullyQualifiedName) { + state.InFlightUid = null; + state.InFlightName = null; + } + + var errorMessage = DecodeOrNull (status.Values, InstrumentationProtocol.KeyMessageBase64); + var stackTrace = DecodeOrNull (status.Values, InstrumentationProtocol.KeyStackBase64); + + var failureMessage = errorMessage ?? "Test failed"; + if (!string.IsNullOrEmpty (stackTrace)) + failureMessage += "\n" + stackTrace; + + // An unrecognized outcome is reported as an error rather than silently + // counted as a pass. + var stateProperty = outcome switch { + InstrumentationProtocol.OutcomePassed => (IProperty) new PassedTestNodeStateProperty (), + InstrumentationProtocol.OutcomeFailed => new FailedTestNodeStateProperty (failureMessage), + InstrumentationProtocol.OutcomeSkipped => new SkippedTestNodeStateProperty (errorMessage), + _ => new ErrorTestNodeStateProperty ($"Unrecognized test outcome '{outcome}'."), + }; + + var properties = new List { stateProperty }; + if (!string.IsNullOrEmpty (className)) + properties.Add (new TrxFullyQualifiedTypeNameProperty (className)); + if (outcome == InstrumentationProtocol.OutcomeFailed && (!string.IsNullOrEmpty (errorMessage) || !string.IsNullOrEmpty (stackTrace))) + properties.Add (new TrxExceptionProperty (errorMessage, stackTrace)); + + var testNode = new TestNode { + Uid = new TestNodeUid (fullyQualifiedName), + DisplayName = displayName ?? fullyQualifiedName, + Properties = new PropertyBag (properties.ToArray ()), + }; + + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, testNode)); + state.ReportedFinal.Add (fullyQualifiedName); + } + + /// + /// Reports a mid-run process crash. If a test was executing when the process + /// died, that test is resolved from "in progress" to an + /// (an infrastructure error, not an assertion failure) so it isn't left + /// dangling and the crash is attributed to it. Otherwise a synthetic error + /// node is published so the overall run is still clearly marked failed. + /// + async Task PublishCrashAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults, StreamingState state) + { + var message = bundleResults.Error ?? "The test process terminated before all tests completed (likely a native crash)."; + Console.Error.WriteLine ($"[AndroidTestAdapter] {message}"); + + TestNode crashNode; + if (state.InFlightUid != null) { + crashNode = new TestNode { + Uid = new TestNodeUid (state.InFlightUid), + DisplayName = state.InFlightName ?? state.InFlightUid, + Properties = new PropertyBag ( + new ErrorTestNodeStateProperty ($"The test process crashed while running this test.\n{message}"), + new TrxExceptionProperty (message, null)), + }; + state.ReportedFinal.Add (state.InFlightUid); + state.InFlightUid = null; + state.InFlightName = null; + } else { + crashNode = new TestNode { + Uid = new TestNodeUid ($"{instrumentation}.ProcessCrashed"), + DisplayName = "Test process crashed before completion", + Properties = new PropertyBag ( + new ErrorTestNodeStateProperty (message), + new TrxFullyQualifiedTypeNameProperty (instrumentation), + new TrxExceptionProperty (message, null)), + }; + } + + await context.MessageBus.PublishAsync (this, new TestNodeUpdateMessage (sessionUid, crashNode)); + } + + /// + /// Fallback path used only when no results were streamed (e.g. an older + /// on-device instrumentation): pull and parse the TRX, then publish all tests. + /// + async Task ReportFromTrxAsync (ExecuteRequestContext context, SessionUid sessionUid, InstrumentationBundleResult bundleResults) + { if (bundleResults.Error != null) { var message = $"Error from instrumentation: {bundleResults.Error}"; if (bundleResults.InstrumentationCode.HasValue) @@ -61,13 +297,6 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU throw new InvalidOperationException (message); } - if (verbose) { - Console.WriteLine ($"[AndroidTestAdapter] Instrumentation results: passed={bundleResults.Passed}, failed={bundleResults.Failed}, skipped={bundleResults.Skipped}"); - if (bundleResults.ResultsPath != null) - Console.WriteLine ($"[AndroidTestAdapter] TRX path on device: {bundleResults.ResultsPath}"); - } - - // 2. Pull and parse TRX if (bundleResults.ResultsPath == null) throw new InvalidOperationException ("Instrumentation did not report a resultsPath in the bundle."); @@ -107,27 +336,34 @@ async Task RunAndReportAsync (ExecuteRequestContext context, SessionUid sessionU } } - async Task RunInstrumentationOnDeviceAsync (CancellationToken cancellationToken) + static string? DecodeOrNull (IReadOnlyDictionary values, string key) { - var cmdArgs = $"shell am instrument -w {package}/{instrumentation}"; - var (exitCode, output, error) = await AdbHelper.RunAsync (adbPath, adbTarget, cmdArgs, cancellationToken, verbose); - - if (verbose) { - Console.WriteLine ($"[AndroidTestAdapter] Exit code: {exitCode}"); - Console.WriteLine (output); + if (!values.TryGetValue (key, out var encoded) || string.IsNullOrEmpty (encoded)) + return null; + try { + return Encoding.UTF8.GetString (Convert.FromBase64String (encoded)); + } catch (FormatException) { + return encoded; } + } - if (exitCode != 0) { - var failureMessage = !string.IsNullOrWhiteSpace (error) ? error : output; - if (string.IsNullOrWhiteSpace (failureMessage)) - failureMessage = $"adb shell am instrument failed with exit code {exitCode}."; - Console.Error.WriteLine ($"[AndroidTestAdapter] {failureMessage}"); - return new InstrumentationBundleResult { - Error = failureMessage, - }; + /// + /// Extracts a human-readable crash reason from the instrumentation output + /// (shortMsg/longMsg are emitted by 'am instrument' when the process crashes). + /// + static string? ExtractCrashMessage (string output) + { + string? shortMsg = null, longMsg = null; + foreach (var rawLine in output.Split ('\n')) { + var line = rawLine.TrimEnd ('\r'); + if (line.StartsWith ("INSTRUMENTATION_RESULT: shortMsg=", StringComparison.Ordinal)) + shortMsg = line.Substring ("INSTRUMENTATION_RESULT: shortMsg=".Length).Trim (); + else if (line.StartsWith ("INSTRUMENTATION_RESULT: longMsg=", StringComparison.Ordinal)) + longMsg = line.Substring ("INSTRUMENTATION_RESULT: longMsg=".Length).Trim (); } - - return ParseInstrumentationBundle (output, error); + if (longMsg != null || shortMsg != null) + return $"The test process crashed: {longMsg ?? shortMsg}"; + return null; } async Task PullTrxFileAsync (string devicePath, CancellationToken cancellationToken) @@ -191,15 +427,15 @@ static InstrumentationBundleResult ParseInstrumentationBundle (string output, st } } - if (bundleValues.TryGetValue ("passed", out var passedStr) && int.TryParse (passedStr, out int passed)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyPassedCount, out var passedStr) && int.TryParse (passedStr, out int passed)) result.Passed = passed; - if (bundleValues.TryGetValue ("failed", out var failedStr) && int.TryParse (failedStr, out int failed)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyFailedCount, out var failedStr) && int.TryParse (failedStr, out int failed)) result.Failed = failed; - if (bundleValues.TryGetValue ("skipped", out var skippedStr) && int.TryParse (skippedStr, out int skipped)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeySkippedCount, out var skippedStr) && int.TryParse (skippedStr, out int skipped)) result.Skipped = skipped; - if (bundleValues.TryGetValue ("resultsPath", out var resultsPath)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyResultsPath, out var resultsPath)) result.ResultsPath = resultsPath; - if (bundleValues.TryGetValue ("error", out var bundleError)) + if (bundleValues.TryGetValue (InstrumentationProtocol.KeyError, out var bundleError)) result.Error = bundleError; // Surface adb stderr if no results were parsed @@ -282,6 +518,130 @@ class InstrumentationBundleResult public string? ResultsPath { get; set; } public string? Error { get; set; } public int? InstrumentationCode { get; set; } + public bool Crashed { get; set; } +} + +/// +/// Mutable state shared across a streamed run: the tests streamed with a final +/// outcome (a non-empty set means streaming was authoritative, so the TRX +/// fallback is skipped), and the test currently executing (if any). +/// +sealed class StreamingState +{ + public HashSet ReportedFinal { get; } = new (StringComparer.Ordinal); + public string? InFlightUid { get; set; } + public string? InFlightName { get; set; } +} + +/// +/// A single completed INSTRUMENTATION_STATUS block: its key/value pairs +/// plus the trailing INSTRUMENTATION_STATUS_CODE. +/// +class InstrumentationStatus +{ + public required IReadOnlyDictionary Values { get; init; } + public int Code { get; init; } +} + +/// +/// A that splits the (arbitrarily chunked) writes it +/// receives from ProcessUtils.StartProcess into complete lines and invokes +/// a callback for each one, so instrumentation output can be parsed as it streams. +/// +sealed class LineWriter (Action onLine) : TextWriter +{ + readonly StringBuilder buffer = new (); + + public override Encoding Encoding => Encoding.UTF8; + + public override void Write (char value) + { + if (value == '\n') + Flush (); + else if (value != '\r') + buffer.Append (value); + } + + public override void Write (char[] buffer, int index, int count) + { + for (int i = 0; i < count; i++) + Write (buffer [index + i]); + } + + public override void Write (string? value) + { + if (value == null) + return; + foreach (var c in value) + Write (c); + } + + // Emit whatever has been buffered as a completed line. + public override void Flush () + { + if (buffer.Length == 0) + return; + var line = buffer.ToString (); + buffer.Clear (); + onLine (line); + } +} + +/// +/// Incrementally parses streamed am instrument -r output. Each completed +/// INSTRUMENTATION_STATUS block (terminated by INSTRUMENTATION_STATUS_CODE) +/// is written to for the consumer to publish, while the +/// raw text is also accumulated for the final results-bundle parse. +/// +sealed class StatusStreamParser (ChannelWriter writer, StringBuilder fullOutput, bool verbose) +{ + const string StatusPrefix = "INSTRUMENTATION_STATUS: "; + const string StatusCodePrefix = "INSTRUMENTATION_STATUS_CODE: "; + + Dictionary? current; + string? lastKey; + + public void OnLine (string line) + { + fullOutput.Append (line).Append ('\n'); + if (verbose) + Console.WriteLine (line); + + if (line.StartsWith (StatusCodePrefix, StringComparison.Ordinal)) { + var codeStr = line.Substring (StatusCodePrefix.Length).Trim (); + int.TryParse (codeStr, out int code); + if (current != null) { + writer.TryWrite (new InstrumentationStatus { Values = current, Code = code }); + current = null; + lastKey = null; + } + return; + } + + if (line.StartsWith (StatusPrefix, StringComparison.Ordinal)) { + var kvp = line.Substring (StatusPrefix.Length); + var eqIndex = kvp.IndexOf ('='); + if (eqIndex > 0) { + current ??= new Dictionary (StringComparer.Ordinal); + lastKey = kvp.Substring (0, eqIndex).Trim (); + current [lastKey] = kvp.Substring (eqIndex + 1); + } + return; + } + + // Continuation of a multi-line value (should be rare — message/stack are + // Base64-encoded on the device precisely to avoid this). + if (current != null && lastKey != null) + current [lastKey] += "\n" + line; + } + + // Drop any block that never received a terminating status code (e.g. the + // process crashed mid-status); it is intentionally not published. + public void Complete () + { + current = null; + lastKey = null; + } } enum TrxOutcome diff --git a/src/Microsoft.Android.Run/InstrumentationProtocol.cs b/src/Microsoft.Android.Run/InstrumentationProtocol.cs new file mode 100644 index 00000000000..52d85679470 --- /dev/null +++ b/src/Microsoft.Android.Run/InstrumentationProtocol.cs @@ -0,0 +1,36 @@ +/// +/// Wire contract for the on-device test instrumentation streaming protocol. The +/// device side (Xamarin.Android.UnitTests.TestInstrumentation) emits +/// INSTRUMENTATION_STATUS/INSTRUMENTATION_RESULT bundles through +/// am instrument -r, and the host side (AndroidTestAdapter) parses +/// them. Values are kept as human-readable strings so the raw instrumentation +/// output stays legible in logcat. This file is source-linked into both the +/// device test runner and the host so the two sides cannot drift out of sync. +/// +static class InstrumentationProtocol +{ + // Keys used in the per-test streaming status blocks. + public const string KeyEvent = "event"; + public const string KeyTest = "test"; + public const string KeyName = "name"; + public const string KeyClass = "class"; + public const string KeyOutcome = "outcome"; + public const string KeyMessageBase64 = "message-b64"; + public const string KeyStackBase64 = "stack-b64"; + + // Keys used in the final results bundle. + public const string KeyPassedCount = "passed"; + public const string KeyFailedCount = "failed"; + public const string KeySkippedCount = "skipped"; + public const string KeyResultsPath = "resultsPath"; + public const string KeyError = "error"; + + // "event" values. + public const string EventStart = "start"; + public const string EventFinish = "finish"; + + // "outcome" values. + public const string OutcomePassed = "passed"; + public const string OutcomeFailed = "failed"; + public const string OutcomeSkipped = "skipped"; +} diff --git a/tests/TestRunner.Core/TestInstrumentation.cs b/tests/TestRunner.Core/TestInstrumentation.cs index f9cf928c426..afe20802de3 100644 --- a/tests/TestRunner.Core/TestInstrumentation.cs +++ b/tests/TestRunner.Core/TestInstrumentation.cs @@ -106,14 +106,14 @@ public override void OnStart () Log.Info (LogTag, $"TRX written to: {trxPath}"); Log.Info (LogTag, $"Results: passed={passed}, failed={failed}, skipped={skipped}"); - bundle.PutInt ("passed", passed); - bundle.PutInt ("failed", failed); - bundle.PutInt ("skipped", skipped); - bundle.PutString ("resultsPath", trxPath); + bundle.PutInt (InstrumentationProtocol.KeyPassedCount, passed); + bundle.PutInt (InstrumentationProtocol.KeyFailedCount, failed); + bundle.PutInt (InstrumentationProtocol.KeySkippedCount, skipped); + bundle.PutString (InstrumentationProtocol.KeyResultsPath, trxPath); Finish (Result.Ok, bundle); } catch (Exception ex) { Log.Error (LogTag, $"Test run failed: {ex}"); - bundle.PutString ("error", ex.ToString ()); + bundle.PutString (InstrumentationProtocol.KeyError, ex.ToString ()); Finish (Result.Canceled, bundle); } } @@ -311,7 +311,31 @@ static void WriteTrxFile (string path, List assemblyResults) } /// - /// Sends test status updates through the instrumentation protocol. + /// Android am instrument status codes emitted on the + /// INSTRUMENTATION_STATUS_CODE line. The values mirror + /// AndroidJUnitRunner's conventions so tools scraping the raw output see + /// familiar codes; the host keys off the explicit event/outcome + /// bundle values rather than these numbers. + /// + enum InstrumentationStatusCode + { + Passed = 0, + Start = 1, + Failed = -2, + Skipped = -3, + } + + /// + /// Streams per-test status updates through the instrumentation protocol + /// (am instrument -r) so the host adapter can report each test to MTP + /// as it finishes. This makes results resilient to a mid-run process crash: + /// every test completed before the crash has already been reported, instead + /// of the whole run being lost because the final TRX was never written. + /// + /// Each SendStatus emits an INSTRUMENTATION_STATUS block that + /// the host parses line-by-line. Because that protocol is line-based, the + /// (potentially multi-line) failure message and stack trace are Base64-encoded + /// so every value stays on a single line. /// class TestListener (Instrumentation instrumentation) : ITestListener { @@ -320,6 +344,13 @@ public void TestStarted (ITest test) if (test.IsSuite) return; Log.Info (LogTag, $"[START] {test.FullName}"); + + var b = new Bundle (); + b.PutString (InstrumentationProtocol.KeyEvent, InstrumentationProtocol.EventStart); + b.PutString (InstrumentationProtocol.KeyTest, test.FullName); + b.PutString (InstrumentationProtocol.KeyName, test.Name); + b.PutString (InstrumentationProtocol.KeyClass, test.ClassName ?? ""); + instrumentation.SendStatus ((Result) (int) InstrumentationStatusCode.Start, b); } public void TestFinished (ITestResult result) @@ -327,20 +358,30 @@ public void TestFinished (ITestResult result) if (result.Test.IsSuite) return; - var outcome = result.ResultState.Status switch { - TestStatus.Passed => "passed", - TestStatus.Failed => "failed", - _ => "skipped", + var (outcome, statusCode) = result.ResultState.Status switch { + TestStatus.Passed => (InstrumentationProtocol.OutcomePassed, InstrumentationStatusCode.Passed), + TestStatus.Failed => (InstrumentationProtocol.OutcomeFailed, InstrumentationStatusCode.Failed), + _ => (InstrumentationProtocol.OutcomeSkipped, InstrumentationStatusCode.Skipped), }; Log.Info (LogTag, $"[{outcome.ToUpperInvariant ()}] {result.FullName}"); var b = new Bundle (); - b.PutString ("test", result.FullName); - b.PutString ("outcome", outcome); - instrumentation.SendStatus (0, b); + b.PutString (InstrumentationProtocol.KeyEvent, InstrumentationProtocol.EventFinish); + b.PutString (InstrumentationProtocol.KeyTest, result.FullName); + b.PutString (InstrumentationProtocol.KeyName, result.Test.Name); + b.PutString (InstrumentationProtocol.KeyClass, result.Test.ClassName ?? ""); + b.PutString (InstrumentationProtocol.KeyOutcome, outcome); + if (result.Message is not null) + b.PutString (InstrumentationProtocol.KeyMessageBase64, Encode (result.Message)); + if (result.StackTrace is not null) + b.PutString (InstrumentationProtocol.KeyStackBase64, Encode (result.StackTrace)); + instrumentation.SendStatus ((Result) (int) statusCode, b); } + static string Encode (string value) + => Convert.ToBase64String (System.Text.Encoding.UTF8.GetBytes (value)); + public void TestOutput (TestOutput output) { } public void SendMessage (TestMessage message) { } } diff --git a/tests/TestRunner.Core/TestRunner.Core.NET.csproj b/tests/TestRunner.Core/TestRunner.Core.NET.csproj index a2090bd5d1b..269fb5a759c 100644 --- a/tests/TestRunner.Core/TestRunner.Core.NET.csproj +++ b/tests/TestRunner.Core/TestRunner.Core.NET.csproj @@ -22,4 +22,10 @@ + + + + +