ci: measure and cut the store-test step (#126)

.github/workflows/azure-objects.yml

@@ -62,13 +62,20 @@ jobs:
           echo "AZURE_FEDERATED_TOKEN_FILE=$RUNNER_TEMP/assertion" >> "$GITHUB_ENV"
 
       - name: Test against the storage account
+        # #126: two contract classes against a real account; the services
+        # leg's limit and hang dump, which run the same classes on emulators.
+        timeout-minutes: 10
         run: >
           dotnet test
           --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj
           --configuration Release
           --no-build
           --filter-class Varve.Store.Tests.Objects.AzureAccountObjectStoreContractTests
           --filter-class Varve.Store.Tests.Objects.AzureAccountStorageContractTests
+          --report-trx
+          --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests
+          --hangdump
+          --hangdump-timeout 5m
 
       - name: What-if for the Container Apps template
         env:
@@ -101,3 +108,13 @@ jobs:
       - name: Remove the token
         if: always()
         run: rm -f "$RUNNER_TEMP/assertion"
+
+      - name: Upload the test results
+        # #126: every run keeps its TRX, pass or fail, so a slow or red run
+        # can be explained test by test from its own artifacts.
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: test-results-azure-objects
+          path: ${{ github.workspace }}/artifacts/test-results
+          if-no-files-found: warn

.github/workflows/nightly.yml

@@ -0,0 +1,105 @@
+# The store's searches at their full depth, every night on main (#126).
+#
+# Some of the store's tests search a space wider than a pull request can wait
+# for. On a pull request each explores a share of it; here, every variable in
+# docs/testing.md section 8 is set, and each explores the whole: as much as
+# the gate explored before #126, or more. The property each test proves is the
+# same at either depth.
+#
+# Not a required check: it runs after the commits it tests have landed. A red
+# night is a defect on main, found the next morning instead of never. #129 may
+# make this run a release's gate.
+#
+# No container: the suites here run in process. The services legs keep their
+# own job in ci.yml and its images in .github/ci-images.yaml.
+name: Nightly
+
+on:
+  schedule:
+    # 02:41 UTC every day: off the hour, as GitHub asks of scheduled workflows.
+    - cron: "41 2 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  DOTNET_NOLOGO: "true"
+  DOTNET_CLI_TELEMETRY_OPTOUT: "true"
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: "true"
+  NUGET_PACKAGES: ${{ github.workspace }}/.nuget/packages
+
+jobs:
+  store:
+    name: store, full depth (${{ matrix.os }})
+    if: github.ref == 'refs/heads/main'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+
+    steps:
+      - name: Line endings
+        # As the build job: the bytes tested are the bytes committed (ADR 0036).
+        shell: bash
+        run: git config --global core.autocrlf false
+
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6
+        with:
+          global-json-file: global.json
+
+      - name: Cache NuGet packages
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: ${{ env.NUGET_PACKAGES }}
+          key: nuget-${{ runner.os }}-${{ hashFiles('Directory.Packages.props') }}
+          restore-keys: nuget-${{ runner.os }}-
+
+      - name: Build
+        run: dotnet build tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release
+
+      - name: Temporary folder on the runner's temporary drive
+        # As the build job (#126): the tests' temporary folder on D:, not C:.
+        if: runner.os == 'Windows'
+        shell: pwsh
+        run: |
+          "TEMP=$env:RUNNER_TEMP" >> $env:GITHUB_ENV
+          "TMP=$env:RUNNER_TEMP" >> $env:GITHUB_ENV
+
+      - name: Test Varve.Store at full depth
+        # The pull request's depth runs in the build job; this is the rest.
+        # Each value covers the whole of its search (docs/testing.md §8). The
+        # crash during recovery at stride 1 runs alone for most of an hour on
+        # Windows with no other test ending, so the hang dump waits longer.
+        timeout-minutes: 150
+        env:
+          VARVE_FAULT_RECOVERY_STRIDE: "1"
+        run: >
+          dotnet test
+          --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj
+          --configuration Release
+          --no-build
+          --report-trx
+          --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests
+          --hangdump
+          --hangdump-timeout 100m
+
+      - name: Store test durations
+        if: always()
+        run: >
+          dotnet run eng/test-durations.cs --
+          ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests
+          --label "Varve.Store.Tests at full depth on ${{ matrix.os }}"
+
+      - name: Upload the test results
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: test-results-nightly-${{ matrix.os }}
+          path: ${{ github.workspace }}/artifacts/test-results
+          if-no-files-found: warn

.github/workflows/publish.yml

@@ -101,43 +101,45 @@ jobs:
         run: dotnet build Varve.slnx --configuration Release --no-restore
 
       - name: Test the analyzers
-        run: dotnet test --project tests/Varve.Analyzers.Tests/Varve.Analyzers.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Analyzers.Tests/Varve.Analyzers.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Analyzers.Tests
 
       - name: Test Varve.Iri
-        run: dotnet test --project tests/Varve.Iri.Tests/Varve.Iri.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Iri.Tests/Varve.Iri.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Iri.Tests
 
       - name: Test Varve.Rdf
-        run: dotnet test --project tests/Varve.Rdf.Tests/Varve.Rdf.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Rdf.Tests/Varve.Rdf.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Rdf.Tests
 
       - name: Test Varve.Turtle
-        run: dotnet test --project tests/Varve.Turtle.Tests/Varve.Turtle.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Turtle.Tests/Varve.Turtle.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Turtle.Tests
 
       - name: Test Varve.RdfXml
-        run: dotnet test --project tests/Varve.RdfXml.Tests/Varve.RdfXml.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.RdfXml.Tests/Varve.RdfXml.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.RdfXml.Tests
 
       - name: Test Varve.JsonLd
-        run: dotnet test --project tests/Varve.JsonLd.Tests/Varve.JsonLd.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.JsonLd.Tests/Varve.JsonLd.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.JsonLd.Tests
 
       - name: Test Varve.Delta
-        run: dotnet test --project tests/Varve.Delta.Tests/Varve.Delta.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Delta.Tests/Varve.Delta.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Delta.Tests
 
       - name: Test Varve.Store
-        run: dotnet test --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release
+        # #126: the build job's Ubuntu limit and hang dump, for the same suite.
+        timeout-minutes: 35
+        run: dotnet test --project tests/Varve.Store.Tests/Varve.Store.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Store.Tests --hangdump --hangdump-timeout 15m
 
       - name: Test Varve.Xsd
-        run: dotnet test --project tests/Varve.Xsd.Tests/Varve.Xsd.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Xsd.Tests/Varve.Xsd.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Xsd.Tests
 
       - name: Test Varve.Sparql
-        run: dotnet test --project tests/Varve.Sparql.Tests/Varve.Sparql.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Sparql.Tests/Varve.Sparql.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Tests
 
       - name: Test Varve.Sparql.Results
-        run: dotnet test --project tests/Varve.Sparql.Results.Tests/Varve.Sparql.Results.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Sparql.Results.Tests/Varve.Sparql.Results.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Results.Tests
 
       - name: Test Varve.Sparql.Evaluation
-        run: dotnet test --project tests/Varve.Sparql.Evaluation.Tests/Varve.Sparql.Evaluation.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Sparql.Evaluation.Tests/Varve.Sparql.Evaluation.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Evaluation.Tests
 
       - name: Test Varve.Sparql.Store
-        run: dotnet test --project tests/Varve.Sparql.Store.Tests/Varve.Sparql.Store.Tests.csproj --configuration Release
+        run: dotnet test --project tests/Varve.Sparql.Store.Tests/Varve.Sparql.Store.Tests.csproj --configuration Release --report-trx --results-directory ${{ github.workspace }}/artifacts/test-results/Varve.Sparql.Store.Tests
 
       - name: Test Varve.Shacl
         run: dotnet test --project tests/Varve.Shacl.Tests/Varve.Shacl.Tests.csproj --configuration Release
@@ -244,6 +246,16 @@ jobs:
           --source https://api.nuget.org/v3/index.json
           --skip-duplicate
 
+      - name: Upload the test results
+        # #126: every run keeps its TRX, pass or fail, so a slow or red run
+        # can be explained test by test from its own artifacts.
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: test-results-publish
+          path: ${{ github.workspace }}/artifacts/test-results
+          if-no-files-found: warn
+
   # The container image (ADR 0111), after the gates: the Native AOT binary
   # published on a native runner per architecture, built into the chiseled
   # image and pushed by digest; the manifest job joins the two, tags the

Directory.Packages.props

@@ -188,6 +188,7 @@
   <ItemGroup Label="Test-time">
     <PackageVersion Include="xunit.v3" Version="4.0.1" Adr="0009" />
     <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.4.1" Adr="0007" />
+    <PackageVersion Include="Microsoft.Testing.Extensions.HangDump" Version="2.4.1" Adr="0007" />
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Analyzer.Testing" Version="1.1.4" Adr="0004" />
     <!-- A transitive dependency of the analyzer harness, pinned rather than
          inherited: Microsoft.CodeAnalysis.Analyzer.Testing asks for

docs/adr/0007-w3c-conformance-harness.md

@@ -183,3 +183,28 @@ maintainer accepts it (ADR 0066).
 
 **The ledger.** No ruling changes; `W3cSuitesPinnedSubmodule` and
 `ConformanceRatchet` cover the new suites as written.
+
+## Amendment, 2026-10-19 — `Microsoft.Testing.Extensions.HangDump`
+
+Filed by #126, unaccepted until the maintainer accepts it (ADR 0066).
+
+- **The need.** No step that runs `Varve.Store.Tests` had a time limit or a
+  dump: a hang held a runner to GitHub's six-hour job limit and named no
+  test. Each such step now has `timeout-minutes`, and below it
+  `--hangdump --hangdump-timeout <t>`, which writes a dump of the test host
+  when no test has reported for `<t>` and names the tests in flight. VSTest's
+  `--blame-hang-timeout` is not available: this repository runs on
+  Microsoft.Testing.Platform (ADR 0006).
+- **The entry.** `Microsoft.Testing.Extensions.HangDump` 2.4.1, the same
+  platform line as `Microsoft.Testing.Extensions.TrxReport` 2.4.1, which this
+  ADR already admits for the same reason: an extension the platform does not
+  build in. Test-time only, referenced by `Varve.Store.Tests` alone, never by
+  anything packable. MIT and managed; it brings
+  `Microsoft.Diagnostics.NETCore.Client` 0.2.607501 (MIT, managed, the
+  runtime's diagnostics client) and `Microsoft.Extensions.Logging.Abstractions`
+  transitively, neither with a native asset.
+- **When it leaves.** When the platform builds hang dumps in, or the store
+  tests stop being the long step a hang would hide in.
+
+**The ledger.** The ruling of this block is `HangDumpForTheStoreTests`,
+unaccepted until the maintainer accepts it.

docs/decisions/w3c-conformance-harness.md

@@ -39,6 +39,8 @@ decisions:
     statement: "eng/ scripts are C# file-based apps run with dotnet run, one implementation for every operating system"
     accepted-by: mailto:emil@okkels-klein.dk
     accepted-at: 2026-09-20T00:00:00Z
+  - key: HangDumpForTheStoreTests
+    statement: "Microsoft.Testing.Extensions.HangDump is a test-time register entry under this ADR, referenced by Varve.Store.Tests alone, so a hang in a store-test step writes a dump naming its tests before the step's time limit; amended 2026-10-19"
 ---
 
 The rulings of [ADR 0007](../adr/0007-w3c-conformance-harness.md) still in force, one line each. The ADR is the
@@ -47,3 +49,5 @@ narrative; this is what code cites. Acceptance is transcribed from the ADR's Sta
 `ManifestsReadByVarveTurtle` is the exit criterion's outcome, recorded by the undated
 "Met at milestone 3b" note inside the ADR; it carries 2026-09-22, the date of the milestone
 3b record, and replaces the temporary dotNetRdf.Core reader the ADR first decided.
+
+`HangDumpForTheStoreTests` is the amendment of 2026-10-19 (#126), filed unaccepted (ADR 0066).

docs/spec/storage-format.md

@@ -49,9 +49,10 @@ storage contract that writes them is ADRs 0018, 0040 and 0071.
     00000001.seg
     …
   derived/
-    .gitignore          "*", written at creation
+    .gitignore          "*", written at open when it is not; not flushed
     LOCK                the lease (ADR 0075), held open; not a blob
-    LOCK.owner          who holds it, for the refusal's message; not a blob
+    LOCK.owner          who holds it, for the refusal's message; not a blob,
+                        not flushed
     checkpoints/<P>
     checkpoints/<P>.graphs   the graph-version index at P (ADR 0125)
     index/state

docs/testing.md

@@ -230,3 +230,32 @@ merges and deletions runs against Azurite and RustFS as well, in the
 are run with merges off (`T.Options()`); the merge suites turn them on and
 drive maintenance by hand, because a second dataset opened over one memory
 storage, which only one process may write, would race a background merge.
+
+## 8. How long the tests take
+
+Every `dotnet test` step in CI writes a TRX and uploads it, pass or fail,
+and the store's steps write their slowest tests, and the classes that held
+the parallel slots, to the job summary (`eng/test-durations.cs`, #126). A
+slow run is explained from its own artifacts; two runs compare test by test.
+Each store step has a time limit at about twice its slowest green run, and a
+hang dump below it that names the tests in flight.
+
+Some searches are wider than a pull request can wait for. Each such test
+explores a pull request's share of its space by default, and the whole of
+it when its variable is set. **The property is the same at either depth**;
+only how much of the space is searched changes, and the pull request's share
+is a subset of the nightly's. Where a share is chosen by a seed, each run
+draws a new one, so that pull requests between them search the whole space;
+the seed starts the test's output and any failure's message, and its
+variable replays the run. The nightly run (`.github/workflows/nightly.yml`,
+on `main`) sets every depth below.
+
+| Variable | Test | Pull request | Nightly |
+|---|---|---:|---:|
+| `VARVE_FAULT_RECOVERY_STRIDE` | `FaultInjectionTests.CrashDuringRecovery.a_crash_during_recovery_recovers`: crashes the workload at every *n*th of its 1,428 operations, then its recovery at every operation | 25, from operation 1 + seed mod 25 (about 11,500 points) | 1 (about 288,000 points; the gate's was 5, 57,679) |
+| `VARVE_FAULT_RECOVERY_SEED` | the same: the seed that chooses where a pull request's share starts | a new one each run | n/a at stride 1 |
+
+The variables of §6 that hunt rather than gate (`VARVE_FAULT_SEEDS`,
+`VARVE_OBJECT_FAULT_SEEDS`, `VARVE_OBJECT_FAULT_STEPS`,
+`VARVE_HTTP_FAULT_STEPS`) stay what they were: neither a pull request nor the
+nightly run sets them.