From 1d543f214d30c29ddf8742035edf421a5ee48d73 Mon Sep 17 00:00:00 2001 From: "Claude (Emil)" Date: Mon, 19 Oct 2026 14:01:51 +0000 Subject: [PATCH 01/14] ci: a TRX from every test step, uploaded, and the store's slowest tests in the summary The Test Varve.Store step takes 27-38 minutes on Windows since #116 and nothing in the log says where: Microsoft.Testing.Platform prints only failed and skipped tests, and no run kept a report. Every dotnet test step in ci.yml, azure-objects.yml and publish.yml now writes a TRX under artifacts/test-results/, and every job that tests uploads that folder, pass or fail. eng/test-durations.cs, BCL only, reads a run's TRX and writes the 20 slowest tests and the classes by busy time, with the minutes into the run at which each class started and finished, to the job summary; the build job runs it after the store step, and storage (services) after its Objects run. Refs #126 Signed-off-by: Emil Okkels Klein Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01YWZaiBxSK7r6aizeG5YWo3 --- .github/workflows/azure-objects.yml | 12 ++ .github/workflows/ci.yml | 176 ++++++++++++++++++++++- .github/workflows/publish.yml | 36 +++-- eng/test-durations.cs | 213 ++++++++++++++++++++++++++++ 4 files changed, 423 insertions(+), 14 deletions(-) create mode 100644 eng/test-durations.cs diff --git a/.github/workflows/azure-objects.yml b/.github/workflows/azure-objects.yml index 46ea731b..74fb52a0 100644 --- a/.github/workflows/azure-objects.yml +++ b/.github/workflows/azure-objects.yml @@ -69,6 +69,8 @@ jobs: --no-build --filter-class Varve.Store.Tests.Objects.AzureAccountObjectStoreContractTests --filter-class Varve.Store.Tests.Objects.AzureAccountStorageContractTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests - name: What-if for the Container Apps template env: @@ -101,3 +103,13 @@ jobs: - name: Remove the token if: always() run: rm -f "$RUNNER_TEMP/assertion" + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-azure-objects + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8acec39c..4a300611 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,96 +259,137 @@ jobs: dotnet test --project tests/Varve.Analyzers.Tests/Varve.Analyzers.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Analyzers.Tests - name: Test Varve.Iri run: > dotnet test --project tests/Varve.Iri.Tests/Varve.Iri.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Iri.Tests - name: Test Varve.Rdf run: > dotnet test --project tests/Varve.Rdf.Tests/Varve.Rdf.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Rdf.Tests - name: Test Varve.Turtle run: > dotnet test --project tests/Varve.Turtle.Tests/Varve.Turtle.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Turtle.Tests - name: Test Varve.RdfXml run: > dotnet test --project tests/Varve.RdfXml.Tests/Varve.RdfXml.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.RdfXml.Tests - name: Test Varve.JsonLd run: > dotnet test --project tests/Varve.JsonLd.Tests/Varve.JsonLd.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.JsonLd.Tests - name: Test Varve.Delta run: > dotnet test --project tests/Varve.Delta.Tests/Varve.Delta.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Delta.Tests - name: Test Varve.Json.Canonical run: > dotnet test --project tests/Varve.Json.Canonical.Tests/Varve.Json.Canonical.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Json.Canonical.Tests - name: Test Varve.Events run: > dotnet test --project tests/Varve.Events.Tests/Varve.Events.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Events.Tests - name: Test Varve.Store run: > dotnet test --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests + + - name: Store test durations + # #126: the slowest tests and the classes that held the parallel slots, + # in the job summary, from the step's own TRX. + if: always() + run: > + dotnet run eng/test-durations.cs -- + ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests + --label "Varve.Store.Tests on ${{ matrix.os }}" - name: Test Varve.Xsd run: > dotnet test --project tests/Varve.Xsd.Tests/Varve.Xsd.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Xsd.Tests - name: Test Varve.Sparql run: > dotnet test --project tests/Varve.Sparql.Tests/Varve.Sparql.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Tests - name: Test Varve.Sparql.Results run: > dotnet test --project tests/Varve.Sparql.Results.Tests/Varve.Sparql.Results.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Results.Tests - name: Test Varve.Sparql.Evaluation run: > dotnet test --project tests/Varve.Sparql.Evaluation.Tests/Varve.Sparql.Evaluation.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Evaluation.Tests - name: Test Varve.Sparql.Store run: > dotnet test --project tests/Varve.Sparql.Store.Tests/Varve.Sparql.Store.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Store.Tests - name: Test Varve.Protocol run: > dotnet test --project tests/Varve.Protocol.Tests/Varve.Protocol.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Protocol.Tests # Built with the solution since 7b and never run by CI until the # Client and integration milestone's second part found the gap. @@ -357,6 +398,8 @@ jobs: dotnet test --project tests/Varve.Protocol.Client.Tests/Varve.Protocol.Client.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Protocol.Client.Tests # Layer (a) of ADR 0100: the permission matrix and every way a token can # be wrong, against an issuer in the test, on every platform. @@ -365,6 +408,8 @@ jobs: dotnet test --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests # tools/repo-standard: its own solution, built and tested on both # platforms (ADR 0039). Not packed: it is not published from here. @@ -376,6 +421,8 @@ jobs: dotnet test --project tools/repo-standard/tests/RepoStandard.Tests/RepoStandard.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/RepoStandard.Tests # The publish dry run. A version pushed to nuget.org cannot be edited, # replaced or deleted, so the pack step and its output are exercised on @@ -392,6 +439,16 @@ jobs: - name: Check the package metadata run: dotnet run eng/package-metadata.cs -- ${{ github.workspace }}/artifacts/packages + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-build-${{ matrix.os }} + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn + # Constraint 2, proven rather than configured. An analyzer only sees what it # can see statically; only an ILC run shows the packages publish, and only # running the binary shows the interesting AOT failures — a trimmed type, a @@ -516,6 +573,18 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ProviderTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-auth-mock-oauth2 + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # The object-storage legs (ADR 0134): the storage contract suite and the # object-store contract, race properties included, against Azurite and the @@ -567,6 +636,15 @@ jobs: --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release --filter-namespace Varve.Store.Tests.Objects + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests + + - name: Store test durations + if: always() + run: > + dotnet run eng/test-durations.cs -- + ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests + --label "Varve.Store.Tests.Objects against the services" # A server's datasets in the container: a second server takes one over, # the first answers writes with writer-superseded, and varve lease @@ -580,6 +658,18 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ObjectStorageServerTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-storage-services + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # The REC generator's bytes on both architectures (ADR 0141): its tests pin # each size's SHA-256, so a pass on each runner is the same bytes on both. @@ -608,7 +698,17 @@ jobs: global-json-file: global.json - name: Generate S, M and L and compare - run: dotnet test --project tests/Varve.Rec.Tests/Varve.Rec.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Rec.Tests/Varve.Rec.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Rec.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-rec-generator-${{ matrix.rid }} + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # Auth test layer (c) (ADR 0100): Zitadel and PostgreSQL, a real provider, # started by compose and seeded through its API by eng/zitadel-seed.cs. One @@ -665,11 +765,23 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ProviderTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests - name: Zitadel's log if: failure() run: docker compose --file .github/zitadel/compose.yaml logs zitadel + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-auth-zitadel + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn + # The real-provider legs (ADR 0100): one test each for Entra ID and Google, # with secrets from the auth-providers environment, which only main can # deploy to — so no pull request, and no fork, ever sees them. Not a required @@ -704,6 +816,18 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ProviderTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-auth-providers + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # The container image (ADR 0111): the Native AOT binary published on a # native runner for its architecture, built into the chiseled image, pushed @@ -894,6 +1018,8 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ProviderTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests - name: Stop the mock and its container if: always() && matrix.rid == 'linux-x64' @@ -956,6 +1082,8 @@ jobs: --project tests/Varve.Server.Tests/Varve.Server.Tests.csproj --configuration Release --filter-class Varve.Server.Tests.ProviderTests + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Server.Tests - name: Stop Zitadel and its container if: always() && matrix.rid == 'linux-x64' @@ -972,6 +1100,16 @@ jobs: path: ${{ github.workspace }}/artifacts/container-protocol if-no-files-found: warn + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-container-${{ matrix.rid }} + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn + # The Aspire sample (ADR 0117): the AppHost starts the server's container # beside the test issuer and its test passes a token, an update and a # query. The image is the one the container job just pushed for this @@ -1018,6 +1156,18 @@ jobs: dotnet test --project samples/Varve.Aspire.Sample.Tests/Varve.Aspire.Sample.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Aspire.Sample.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-aspire + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # Durability, measured per platform (ADR 0073): what an append and a flush # to the device cost on each runner, with the calls the file backend makes. @@ -1073,6 +1223,18 @@ jobs: dotnet test --project samples/Varve.OnBehalfOf.Sample.Tests/Varve.OnBehalfOf.Sample.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.OnBehalfOf.Sample.Tests + + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-samples + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn # The Event Hubs bridge sample (ADRs 0143, 0146): the emulator and Azurite # started by compose, and the sample's test, which runs Varve as a process, @@ -1114,6 +1276,8 @@ jobs: dotnet test --project samples/Varve.EventHubs.Sample.Tests/Varve.EventHubs.Sample.Tests.csproj --configuration Release + --report-trx + --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.EventHubs.Sample.Tests - name: The emulator's log if: failure() @@ -1123,6 +1287,16 @@ jobs: if: always() run: docker compose --file .github/eventhubs/compose.yaml down || true + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-samples-eventhubs + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn + # The Entra registrations of the on-behalf-of chain build (ADR 0127): the # Bicep CLI the runner's Azure CLI carries, and the Microsoft Graph # extension from mcr.microsoft.com. Nothing is deployed. diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 37544d1a..972b93ca 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -101,43 +101,43 @@ jobs: run: dotnet build Varve.slnx --configuration Release --no-restore - name: Test the analyzers - run: dotnet test --project tests/Varve.Analyzers.Tests/Varve.Analyzers.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Analyzers.Tests/Varve.Analyzers.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Analyzers.Tests - name: Test Varve.Iri - run: dotnet test --project tests/Varve.Iri.Tests/Varve.Iri.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Iri.Tests/Varve.Iri.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Iri.Tests - name: Test Varve.Rdf - run: dotnet test --project tests/Varve.Rdf.Tests/Varve.Rdf.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Rdf.Tests/Varve.Rdf.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Rdf.Tests - name: Test Varve.Turtle - run: dotnet test --project tests/Varve.Turtle.Tests/Varve.Turtle.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Turtle.Tests/Varve.Turtle.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Turtle.Tests - name: Test Varve.RdfXml - run: dotnet test --project tests/Varve.RdfXml.Tests/Varve.RdfXml.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.RdfXml.Tests/Varve.RdfXml.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.RdfXml.Tests - name: Test Varve.JsonLd - run: dotnet test --project tests/Varve.JsonLd.Tests/Varve.JsonLd.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.JsonLd.Tests/Varve.JsonLd.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.JsonLd.Tests - name: Test Varve.Delta - run: dotnet test --project tests/Varve.Delta.Tests/Varve.Delta.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Delta.Tests/Varve.Delta.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Delta.Tests - name: Test Varve.Store - run: dotnet test --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests - name: Test Varve.Xsd - run: dotnet test --project tests/Varve.Xsd.Tests/Varve.Xsd.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Xsd.Tests/Varve.Xsd.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Xsd.Tests - name: Test Varve.Sparql - run: dotnet test --project tests/Varve.Sparql.Tests/Varve.Sparql.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Sparql.Tests/Varve.Sparql.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Tests - name: Test Varve.Sparql.Results - run: dotnet test --project tests/Varve.Sparql.Results.Tests/Varve.Sparql.Results.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Sparql.Results.Tests/Varve.Sparql.Results.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Results.Tests - name: Test Varve.Sparql.Evaluation - run: dotnet test --project tests/Varve.Sparql.Evaluation.Tests/Varve.Sparql.Evaluation.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Sparql.Evaluation.Tests/Varve.Sparql.Evaluation.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Evaluation.Tests - name: Test Varve.Sparql.Store - run: dotnet test --project tests/Varve.Sparql.Store.Tests/Varve.Sparql.Store.Tests.csproj --configuration Release + run: dotnet test --project tests/Varve.Sparql.Store.Tests/Varve.Sparql.Store.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Store.Tests - name: Run the W3C suites continue-on-error: true @@ -238,6 +238,16 @@ jobs: --source https://api.nuget.org/v3/index.json --skip-duplicate + - name: Upload the test results + # #126: every run keeps its TRX, pass or fail, so a slow or red run + # can be explained test by test from its own artifacts. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: test-results-publish + path: ${{ github.workspace }}/artifacts/test-results + if-no-files-found: warn + # The container image (ADR 0111), after the gates: the Native AOT binary # published on a native runner per architecture, built into the chiseled # image and pushed by digest; the manifest job joins the two, tags the diff --git a/eng/test-durations.cs b/eng/test-durations.cs new file mode 100644 index 00000000..32a5d058 --- /dev/null +++ b/eng/test-durations.cs @@ -0,0 +1,213 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +// Test durations from a run's TRX. +// +// dotnet run eng/test-durations.cs -- [--top ] [--label ] +// +// Writes the slowest tests of a run, and the classes that held the run's +// parallel slots, to standard output and to the job summary. #126: the store +// suite's step went from 8 to 27–54 minutes on Windows, and nothing in the log +// said where the time went, because Microsoft.Testing.Platform prints only +// failed and skipped tests. +// +// Two tables: +// - the slowest tests (default 20), by their own duration; +// - the slowest classes by their busy time (the sum of their tests' +// durations), with the minutes into the run at which each class's first +// test started and its last test ended. A class whose last test ends long +// after every other class's is the tail the step waits for. +// +// It reads every *.trx under a directory, so a step that writes several (a +// re-run, a filter run twice) is read as one run. +// +// Exit codes: 0 written, 2 could not run. No TRX at all is reported and exits +// 0: the step that should have written it has already failed, and a second red +// step would say nothing new. +// +// See docs/testing.md, "How long the tests take". + +using System.Globalization; +using System.Text; +using System.Xml.Linq; + +XNamespace trxNs = "http://microsoft.com/schemas/VisualStudio/TeamTest/2010"; + +if (args.Length == 0) +{ + Console.Error.WriteLine("usage: dotnet run eng/test-durations.cs -- [--top ] [--label ]"); + return 2; +} + +string input = args[0]; +int top = 20; +string? label = null; +for (int i = 1; i < args.Length - 1; i++) +{ + switch (args[i]) + { + case "--top" when int.TryParse(args[i + 1], NumberStyles.None, CultureInfo.InvariantCulture, out int n) && n > 0: + top = n; + break; + case "--label": + label = args[i + 1]; + break; + default: + break; + } +} + +string[] files = File.Exists(input) + ? [input] + : Directory.Exists(input) + ? Directory.GetFiles(input, "*.trx", SearchOption.AllDirectories) + : []; + +var output = new StringBuilder(); +output.AppendLine(CultureInfo.InvariantCulture, $"### Test durations{(label is null ? "" : $" — {label}")}"); +output.AppendLine(); + +if (files.Length == 0) +{ + output.AppendLine(CultureInfo.InvariantCulture, $"No TRX under `{input}`: the test step did not write one."); + Emit(output.ToString()); + return 0; +} + +var results = new List(); +foreach (string file in files) +{ + XDocument document; + try + { + document = XDocument.Load(file); + } + catch (System.Xml.XmlException e) + { + Console.Error.WriteLine($"{file}: not a TRX ({e.Message})"); + return 2; + } + + // The class is on the definition, the duration on the result; testId joins them. + var classes = new Dictionary(StringComparer.Ordinal); + foreach (XElement test in document.Descendants(trxNs + "UnitTest")) + { + string? id = (string?)test.Attribute("id"); + string? className = (string?)test.Element(trxNs + "TestMethod")?.Attribute("className"); + if (id is not null && className is not null) + { + classes[id] = className; + } + } + + foreach (XElement result in document.Descendants(trxNs + "UnitTestResult")) + { + string name = (string?)result.Attribute("testName") ?? "?"; + string id = (string?)result.Attribute("testId") ?? ""; + string outcome = (string?)result.Attribute("outcome") ?? "?"; + TimeSpan duration = TimeSpan.TryParse((string?)result.Attribute("duration"), CultureInfo.InvariantCulture, out TimeSpan d) ? d : TimeSpan.Zero; + DateTimeOffset? start = DateTimeOffset.TryParse((string?)result.Attribute("startTime"), CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTimeOffset s) ? s : null; + DateTimeOffset? end = DateTimeOffset.TryParse((string?)result.Attribute("endTime"), CultureInfo.InvariantCulture, DateTimeStyles.None, out DateTimeOffset e) ? e : null; + string className = classes.TryGetValue(id, out string? c) ? c : "?"; + results.Add(new Result(className, name, outcome, duration, start, end)); + } +} + +// The namespace every class shares is dropped from the tables, so a suite's +// classes read as SmokeTests and Faults.FaultInjectionTests. +string sharedNamespace = SharedNamespace(results.Select(r => r.Class).Where(c => c != "?").Distinct(StringComparer.Ordinal).ToList()); + +DateTimeOffset? runStart = results.Where(r => r.Start is not null).Select(r => r.Start).Min(); +DateTimeOffset? runEnd = results.Where(r => r.End is not null).Select(r => r.End).Max(); +TimeSpan busy = results.Aggregate(TimeSpan.Zero, (sum, r) => sum + r.Duration); + +output.AppendLine(CultureInfo.InvariantCulture, + $"{results.Count} results from {files.Length} TRX; first start to last end {Minutes(runEnd - runStart)}; busy time (sum of durations) {Minutes(busy)}."); +output.AppendLine(); +output.AppendLine("| # | Duration | Outcome | Class | Test |"); +output.AppendLine("|---:|---:|---|---|---|"); +int rank = 0; +foreach (Result r in results.OrderByDescending(r => r.Duration).Take(top)) +{ + output.AppendLine(CultureInfo.InvariantCulture, + $"| {++rank} | {Minutes(r.Duration)} | {r.Outcome} | {Cell(Short(r.Class))} | {Cell(r.Name)} |"); +} + +output.AppendLine(); +output.AppendLine("Classes by busy time; *from* and *to* are minutes into the run of the class's first start and last end."); +output.AppendLine(); +output.AppendLine("| Class | Tests | Busy | From | To |"); +output.AppendLine("|---|---:|---:|---:|---:|"); +foreach (IGrouping group in results + .GroupBy(r => r.Class, StringComparer.Ordinal) + .OrderByDescending(g => g.Sum(r => r.Duration.Ticks)) + .Take(top)) +{ + TimeSpan classBusy = TimeSpan.FromTicks(group.Sum(r => r.Duration.Ticks)); + DateTimeOffset? from = group.Where(r => r.Start is not null).Select(r => r.Start).Min(); + DateTimeOffset? to = group.Where(r => r.End is not null).Select(r => r.End).Max(); + output.AppendLine(CultureInfo.InvariantCulture, + $"| {Cell(Short(group.Key))} | {group.Count()} | {Minutes(classBusy)} | {Offset(from)} | {Offset(to)} |"); +} + +Emit(output.ToString()); +return 0; + +string Offset(DateTimeOffset? at) => + at is { } value && runStart is { } origin + ? (value - origin).TotalMinutes.ToString("0.0", CultureInfo.InvariantCulture) + : "?"; + +static string Minutes(TimeSpan? span) => + span is not { } value ? "?" + : value.TotalMinutes >= 1 + ? string.Create(CultureInfo.InvariantCulture, $"{(int)value.TotalMinutes}m {value.Seconds:00}s") + : string.Create(CultureInfo.InvariantCulture, $"{value.TotalSeconds:0.0}s"); + +string Short(string className) => + sharedNamespace.Length > 0 && className.StartsWith(sharedNamespace, StringComparison.Ordinal) + ? className[sharedNamespace.Length..] + : className; + +// The longest run of whole namespace segments, with its trailing dot, that +// every class's namespace starts with. +static string SharedNamespace(List classes) +{ + if (classes.Count == 0) + { + return ""; + } + + string[] shared = classes[0].Split('.')[..^1]; + foreach (string className in classes.Skip(1)) + { + string[] parts = className.Split('.')[..^1]; + int same = 0; + while (same < shared.Length && same < parts.Length && shared[same] == parts[same]) + { + same++; + } + + shared = shared[..same]; + } + + return shared.Length == 0 ? "" : string.Join('.', shared) + "."; +} + +static string Cell(string text) +{ + string escaped = text.Replace("|", "\\|", StringComparison.Ordinal).Replace("\n", " ", StringComparison.Ordinal); + return escaped.Length <= 160 ? escaped : string.Concat(escaped.AsSpan(0, 157), "..."); +} + +static void Emit(string text) +{ + Console.Write(text); + if (Environment.GetEnvironmentVariable("GITHUB_STEP_SUMMARY") is { Length: > 0 } summary) + { + File.AppendAllText(summary, text + Environment.NewLine); + } +} + +sealed record Result(string Class, string Name, string Outcome, TimeSpan Duration, DateTimeOffset? Start, DateTimeOffset? End); From 456a263f38ea841cb34f21d3082d19424774eeed Mon Sep 17 00:00:00 2001 From: "Claude (Emil)" Date: Mon, 19 Oct 2026 14:04:09 +0000 Subject: [PATCH 02/14] build: Microsoft.Testing.Extensions.HangDump for the store tests, under ADR 0007 No step that runs Varve.Store.Tests can name a hung test: the platform prints nothing until a test ends, and VSTest's --blame-hang-timeout is not available on Microsoft.Testing.Platform. The HangDump extension writes a dump, and the tests in flight, when no test has reported for a set time. A test-time register entry under ADR 0007, which admits TrxReport for the same reason, by a dated amendment with its ruling HangDumpForTheStoreTests, filed unaccepted (ADR 0066). 2.4.1, TrxReport's platform line; MIT and managed, as is Microsoft.Diagnostics.NETCore.Client, which it brings. Referenced by Varve.Store.Tests alone. The workflow flags follow with the time limits, sized from the measured run. Refs #126 Signed-off-by: Emil Okkels Klein Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01YWZaiBxSK7r6aizeG5YWo3 --- Directory.Packages.props | 1 + docs/adr/0007-w3c-conformance-harness.md | 25 +++++++++++++++++++ docs/decisions/w3c-conformance-harness.md | 4 +++ .../Varve.Store.Tests.csproj | 1 + 4 files changed, 31 insertions(+) diff --git a/Directory.Packages.props b/Directory.Packages.props index c12de078..f52910eb 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -185,6 +185,7 @@ +