Stop verification at the first unanswered request

docs/articles/verify.md

@@ -15,7 +15,9 @@ A verification needs only the serial port of the device.
 dotnet harp.toolkit verify --port COM3
 ```
 
-Every check reports as passed, failed, skipped or error. A failed check ran to completion and the device did not behave as required by the specification. An error means the check could not be completed at all, which happens when the device replies with an error or stays silent. A check is skipped when it needs an option that was not supplied, and the message names the option. A silent register costs one result after a fixed 2000 ms, instead of stalling the rest of the run.
+Every check reports as passed, failed, skipped or error. A failed check ran to completion and the device did not behave as required by the specification. An error means the check could not be completed at all, which happens when the device replies with an error. A check is skipped when it needs an option that was not supplied, and the message names the option.
+
+Verification stops at the first request left unanswered for 2000 ms, since a late reply can be matched to a later check, and the console states where the run stopped. The HTML report is still written when `--report` is supplied, and states that the verification is incomplete.
 
 #### Serial port
 ```ps1
@@ -33,7 +35,7 @@ Prints a detailed result for every check once the run finishes, including the st
 
 ### Exit code
 
-The command exits 1 if any check failed or ended in error, and 0 otherwise. Skipped checks do not affect the result, so a run that skips every optional check still exits 0. A run that cannot start also exits 1, for example when the named serial port is not present.
+The command exits 1 if any check failed or ended in error, and 0 otherwise. Skipped checks do not affect the result, so a run that skips every optional check still exits 0. A run that cannot start or continue also exits 1, for example when the named serial port is not present or the device stops responding.
 
 ## Specification version
 

src/Harp.Toolkit/Verify/Report.cs

@@ -10,6 +10,7 @@ public class Report
     public DateTime RunDate { get; set; } = DateTime.Now;
     public bool IncludePrerelease { get; set; }
     public string ProtocolNotice { get; set; } = string.Empty;
+    public string AbortReason { get; set; } = string.Empty;
     public string DeclaredProtocolVersion { get; set; } = string.Empty;
     public string CheckedProtocolVersion { get; set; } = string.Empty;
     public string ProtocolCommit { get; set; } = string.Empty;

src/Harp.Toolkit/Verify/ReportTemplate.cshtml

@@ -54,6 +54,13 @@
             </div>
         </div>
 
+        @if (Model.AbortReason.Length > 0)
+        {
+            <div class="alert alert-danger" role="alert">
+                <strong>Verification incomplete.</strong> @Model.AbortReason
+            </div>
+        }
+
         @if (Model.ProtocolNotice.Length > 0)
         {
             <div class="alert @(Model.IncludePrerelease ? "alert-info" : "alert-warning")" role="alert">

src/Harp.Toolkit/Verify/Runner.cs

@@ -27,6 +27,11 @@ public IEnumerable<Suite> CollectSuites()
         {
             await foreach (var result in suite.RunAllAsync(connection, includePrerelease, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc)))
             {
+                if (result.Result is ErrorResult { Exception: TimeoutException timeout })
+                {
+                    throw timeout;
+                }
+
                 yield return (suite, result);
             }
         }

src/Harp.Toolkit/Verify/VerifyCommand.cs

@@ -130,30 +130,40 @@ static async Task<int> RunVerification(string portName, FileInfo? reportFile, bo
         };
 
         int currentTest = 0;
-        await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) =>
+        try
         {
-            // Print "Running" status before test execution (without newline)
-            currentTest++;
-            if (!Console.IsOutputRedirected)
-                Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running...");
-        }))
-        {
-            // Clear the line by moving cursor to start and overwriting with spaces, then print result
-            if (!Console.IsOutputRedirected)
-                Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
-            AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}");
-
-            var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name);
-            if (suiteResult == null)
+            await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) =>
             {
-                suiteResult = new SuiteResult
+                // Print "Running" status before test execution (without newline)
+                currentTest++;
+                if (!Console.IsOutputRedirected)
+                    Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running...");
+            }))
+            {
+                // Clear the line by moving cursor to start and overwriting with spaces, then print result
+                if (!Console.IsOutputRedirected)
+                    Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
+                AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}");
+
+                var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name);
+                if (suiteResult == null)
                 {
-                    Name = suite.GetType().Name,
-                    Description = suite.Description
-                };
-                report.Suites.Add(suiteResult);
+                    suiteResult = new SuiteResult
+                    {
+                        Name = suite.GetType().Name,
+                        Description = suite.Description
+                    };
+                    report.Suites.Add(suiteResult);
+                }
+                suiteResult.Results.Add(result);
             }
-            suiteResult.Results.Add(result);
+        }
+        catch (TimeoutException ex)
+        {
+            if (!Console.IsOutputRedirected)
+                Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
+            report.AbortReason = DescribeAbort(ex, currentTest, runner.TestCount);
+            AnsiConsole.MarkupLine($"[red]{Markup.Escape(report.AbortReason)}[/]");
         }
 
         if (verbose)
@@ -215,12 +225,21 @@ static async Task<int> RunVerification(string portName, FileInfo? reportFile, bo
             AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]");
         }
 
+        if (report.AbortReason.Length > 0)
+            return 1;
+
         var failedCount = report.Suites
             .SelectMany(suite => suite.Results)
             .Count(result => result.Result.Status is Status.Failed or Status.Error);
         return failedCount > 0 ? 1 : 0;
     }
 
+    static string DescribeAbort(TimeoutException exception, int completedCount, int testCount)
+    {
+        return $"{exception.Message} Verification stopped after {completedCount} of {testCount} checks. " +
+            "Rerun the verification once the device and the connection are stable.";
+    }
+
     static string GetDeclaredVersion(ProtocolTarget target)
     {
         return target.DeclaredVersion.HasValue

src/Harp.Toolkit/Verify/VerifyConnection.cs

@@ -288,7 +288,7 @@ internal async Task<DeviceIdentity> ReadDeviceIdentityAsync(CancellationToken ca
         {
             return await read(cancellationToken);
         }
-        catch (Exception) when (!cancellationToken.IsCancellationRequested)
+        catch (Exception ex) when (ex is not TimeoutException && !cancellationToken.IsCancellationRequested)
         {
             return null;
         }