diff --git a/docs/articles/verify.md b/docs/articles/verify.md
index 3ae25d8..75735c4 100644
--- a/docs/articles/verify.md
+++ b/docs/articles/verify.md
@@ -15,7 +15,9 @@ A verification needs only the serial port of the device.
dotnet harp.toolkit verify --port COM3
```
-Every check reports as passed, failed, skipped or error. A failed check ran to completion and the device did not behave as required by the specification. An error means the check could not be completed at all, which happens when the device replies with an error or stays silent. A check is skipped when it needs an option that was not supplied, and the message names the option. A silent register costs one result after a fixed 2000 ms, instead of stalling the rest of the run.
+Every check reports as passed, failed, skipped or error. A failed check ran to completion and the device did not behave as required by the specification. An error means the check could not be completed at all, which happens when the device replies with an error. A check is skipped when it needs an option that was not supplied, and the message names the option.
+
+Verification stops at the first request left unanswered for 2000 ms, since a late reply can be matched to a later check, and the console states where the run stopped. The HTML report is still written when `--report` is supplied, and states that the verification is incomplete.
#### Serial port
```ps1
@@ -33,7 +35,7 @@ Prints a detailed result for every check once the run finishes, including the st
### Exit code
-The command exits 1 if any check failed or ended in error, and 0 otherwise. Skipped checks do not affect the result, so a run that skips every optional check still exits 0. A run that cannot start also exits 1, for example when the named serial port is not present.
+The command exits 1 if any check failed or ended in error, and 0 otherwise. Skipped checks do not affect the result, so a run that skips every optional check still exits 0. A run that cannot start or continue also exits 1, for example when the named serial port is not present or the device stops responding.
## Specification version
diff --git a/src/Harp.Toolkit/Verify/Report.cs b/src/Harp.Toolkit/Verify/Report.cs
index 608d991..1bd5612 100644
--- a/src/Harp.Toolkit/Verify/Report.cs
+++ b/src/Harp.Toolkit/Verify/Report.cs
@@ -10,6 +10,7 @@ public class Report
public DateTime RunDate { get; set; } = DateTime.Now;
public bool IncludePrerelease { get; set; }
public string ProtocolNotice { get; set; } = string.Empty;
+ public string AbortReason { get; set; } = string.Empty;
public string DeclaredProtocolVersion { get; set; } = string.Empty;
public string CheckedProtocolVersion { get; set; } = string.Empty;
public string ProtocolCommit { get; set; } = string.Empty;
diff --git a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml
index eae150b..6085409 100644
--- a/src/Harp.Toolkit/Verify/ReportTemplate.cshtml
+++ b/src/Harp.Toolkit/Verify/ReportTemplate.cshtml
@@ -54,6 +54,13 @@
+ @if (Model.AbortReason.Length > 0)
+ {
+
+ Verification incomplete. @Model.AbortReason
+
+ }
+
@if (Model.ProtocolNotice.Length > 0)
{
diff --git a/src/Harp.Toolkit/Verify/Runner.cs b/src/Harp.Toolkit/Verify/Runner.cs
index ed5ca46..9a61489 100644
--- a/src/Harp.Toolkit/Verify/Runner.cs
+++ b/src/Harp.Toolkit/Verify/Runner.cs
@@ -27,6 +27,11 @@ public IEnumerable CollectSuites()
{
await foreach (var result in suite.RunAllAsync(connection, includePrerelease, cancellationToken, (testName, testDesc) => onTestStart?.Invoke(suite, testName, testDesc)))
{
+ if (result.Result is ErrorResult { Exception: TimeoutException timeout })
+ {
+ throw timeout;
+ }
+
yield return (suite, result);
}
}
diff --git a/src/Harp.Toolkit/Verify/VerifyCommand.cs b/src/Harp.Toolkit/Verify/VerifyCommand.cs
index bed848b..a529d74 100644
--- a/src/Harp.Toolkit/Verify/VerifyCommand.cs
+++ b/src/Harp.Toolkit/Verify/VerifyCommand.cs
@@ -130,30 +130,40 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bo
};
int currentTest = 0;
- await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) =>
+ try
{
- // Print "Running" status before test execution (without newline)
- currentTest++;
- if (!Console.IsOutputRedirected)
- Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running...");
- }))
- {
- // Clear the line by moving cursor to start and overwriting with spaces, then print result
- if (!Console.IsOutputRedirected)
- Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
- AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}");
-
- var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name);
- if (suiteResult == null)
+ await foreach (var (suite, result) in runner.RunAllAsync(connection, cancellationToken, (suite, testName, testDesc) =>
{
- suiteResult = new SuiteResult
+ // Print "Running" status before test execution (without newline)
+ currentTest++;
+ if (!Console.IsOutputRedirected)
+ Console.Write($"({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{testName} .... Running...");
+ }))
+ {
+ // Clear the line by moving cursor to start and overwriting with spaces, then print result
+ if (!Console.IsOutputRedirected)
+ Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
+ AnsiConsole.MarkupLine($"[grey]({currentTest}/{runner.TestCount}) {suite.GetType().Name}::{result.Name}[/] .... {GetResultMarkup(result.Result)}");
+
+ var suiteResult = report.Suites.FirstOrDefault(s => s.Name == suite.GetType().Name);
+ if (suiteResult == null)
{
- Name = suite.GetType().Name,
- Description = suite.Description
- };
- report.Suites.Add(suiteResult);
+ suiteResult = new SuiteResult
+ {
+ Name = suite.GetType().Name,
+ Description = suite.Description
+ };
+ report.Suites.Add(suiteResult);
+ }
+ suiteResult.Results.Add(result);
}
- suiteResult.Results.Add(result);
+ }
+ catch (TimeoutException ex)
+ {
+ if (!Console.IsOutputRedirected)
+ Console.Write($"\r{new string(' ', Console.WindowWidth - 1)}\r");
+ report.AbortReason = DescribeAbort(ex, currentTest, runner.TestCount);
+ AnsiConsole.MarkupLine($"[red]{Markup.Escape(report.AbortReason)}[/]");
}
if (verbose)
@@ -215,12 +225,21 @@ static async Task RunVerification(string portName, FileInfo? reportFile, bo
AnsiConsole.MarkupLine($"[green]Done![/] Report generated: [link]{fileName}[/]");
}
+ if (report.AbortReason.Length > 0)
+ return 1;
+
var failedCount = report.Suites
.SelectMany(suite => suite.Results)
.Count(result => result.Result.Status is Status.Failed or Status.Error);
return failedCount > 0 ? 1 : 0;
}
+ static string DescribeAbort(TimeoutException exception, int completedCount, int testCount)
+ {
+ return $"{exception.Message} Verification stopped after {completedCount} of {testCount} checks. " +
+ "Rerun the verification once the device and the connection are stable.";
+ }
+
static string GetDeclaredVersion(ProtocolTarget target)
{
return target.DeclaredVersion.HasValue
diff --git a/src/Harp.Toolkit/Verify/VerifyConnection.cs b/src/Harp.Toolkit/Verify/VerifyConnection.cs
index 2b33da4..179f003 100644
--- a/src/Harp.Toolkit/Verify/VerifyConnection.cs
+++ b/src/Harp.Toolkit/Verify/VerifyConnection.cs
@@ -288,7 +288,7 @@ internal async Task ReadDeviceIdentityAsync(CancellationToken ca
{
return await read(cancellationToken);
}
- catch (Exception) when (!cancellationToken.IsCancellationRequested)
+ catch (Exception ex) when (ex is not TimeoutException && !cancellationToken.IsCancellationRequested)
{
return null;
}