diff --git a/GitBranchStateCache.Tests/Mirrors/MirrorMaintenanceServiceTests.cs b/GitBranchStateCache.Tests/Mirrors/MirrorMaintenanceServiceTests.cs index 4462271..282dd8a 100644 --- a/GitBranchStateCache.Tests/Mirrors/MirrorMaintenanceServiceTests.cs +++ b/GitBranchStateCache.Tests/Mirrors/MirrorMaintenanceServiceTests.cs @@ -288,4 +288,115 @@ public void Sweep_LeavesTheMirrorsThatAreStillWanted() Assert.IsFalse(store.Exists(idle)); Assert.IsTrue(store.Exists(busy)); } + + /// + /// Leaves a clone's staging directory beside where its mirror would go, dated as the clone started. + /// + private static string SeedStaging(MirrorStore store, MockFileSystem fileSystem, string repositoryPath, DateTimeOffset startedAt) + { + store.TryResolve(new MirrorKey("github", repositoryPath), out string? directory); + string staging = $"{directory}.tmp-{Guid.NewGuid():N}"; + fileSystem.Directory.CreateDirectory(fileSystem.Path.Combine(staging, "objects")); + fileSystem.File.WriteAllText(fileSystem.Path.Combine(staging, "objects", "pack"), "partial clone"); + fileSystem.Directory.SetCreationTimeUtc(staging, startedAt.UtcDateTime); + fileSystem.Directory.SetLastWriteTimeUtc(staging, startedAt.UtcDateTime); + return staging; + } + + [TestMethod] + public void Sweep_AStagingDirectoryLeftByADeadClone_IsRemoved() + { + // A process killed mid-clone never runs the finally that discards its staging directory, and + // nothing else would ever look for it. + (MirrorMaintenanceService service, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = + Build(); + + string staging = SeedStaging(store, fileSystem, "studio/game.git", time.GetUtcNow()); + time.Advance(TimeSpan.FromDays(1)); + + service.Sweep(); + + Assert.IsFalse(fileSystem.Directory.Exists(staging)); + } + + [TestMethod] + public void Sweep_WithReapingDisabled_StillRemovesADeadClonesStagingDirectory() + { + // Keeping every mirror is a choice about mirrors. A staging directory is not one. + (MirrorMaintenanceService service, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = + Build(TimeSpan.Zero); + + string staging = SeedStaging(store, fileSystem, "studio/game.git", time.GetUtcNow()); + time.Advance(TimeSpan.FromDays(1)); + + service.Sweep(); + + Assert.IsFalse(fileSystem.Directory.Exists(staging)); + } + + [TestMethod] + public void Sweep_AStagingDirectoryOfACloneStillRunning_IsKept() + { + // A clone may run for the whole of FetchTimeout, so its staging directory is live until then. + (MirrorMaintenanceService service, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = + Build(); + + string staging = SeedStaging(store, fileSystem, "studio/game.git", time.GetUtcNow()); + time.Advance(new GitBranchStateCacheOptions().FetchTimeout); + + service.Sweep(); + + Assert.IsTrue(fileSystem.Directory.Exists(staging)); + } + + [TestMethod] + public void Sweep_ARepositoryPathSegmentThatLooksLikeStaging_IsKept() + { + // Only the exact name a clone gives its staging directory counts. A repository path segment + // that merely starts the same way holds a live mirror. + (MirrorMaintenanceService service, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = + Build(); + + string directory = Seed(store, fileSystem, "studio/mirror.git.tmp-old/game.git"); + store.MarkUsed(directory); + time.Advance(TimeSpan.FromDays(1)); + store.MarkUsed(directory); + + service.Sweep(); + + Assert.IsTrue(store.Exists(directory)); + } + + [TestMethod] + public void EnumerateStaging_FindsOnlyCloneStagingDirectories() + { + (_, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = Build(); + + string staging = SeedStaging(store, fileSystem, "studio/game.git", time.GetUtcNow()); + Seed(store, fileSystem, "studio/tools.git"); + Seed(store, fileSystem, "studio/mirror.git.tmp-notaguid/game.git"); + + CollectionAssert.AreEqual(new[] { staging }, store.EnumerateStaging().ToArray()); + } + + [TestMethod] + public void Sweep_AStagingNamedDirectoryHoldingAMirror_IsKept() + { + // However unlikely the name, a directory with a mirror inside it is a repository path segment, + // and deleting it would take a live mirror with it. + (MirrorMaintenanceService service, MirrorStore store, MockFileSystem fileSystem, FakeTimeProvider time) = + Build(); + + string segment = $"mirror.git.tmp-{Guid.NewGuid():N}"; + string directory = Seed(store, fileSystem, $"studio/{segment}/game.git"); + string parent = fileSystem.Path.GetDirectoryName(directory)!; + fileSystem.Directory.SetCreationTimeUtc(parent, time.GetUtcNow().UtcDateTime); + fileSystem.Directory.SetLastWriteTimeUtc(parent, time.GetUtcNow().UtcDateTime); + time.Advance(TimeSpan.FromDays(1)); + store.MarkUsed(directory); + + service.Sweep(); + + Assert.IsTrue(store.Exists(directory)); + } } diff --git a/GitBranchStateCache/Mirrors/IMirrorStore.cs b/GitBranchStateCache/Mirrors/IMirrorStore.cs index 3fb0eee..5eea4d0 100644 --- a/GitBranchStateCache/Mirrors/IMirrorStore.cs +++ b/GitBranchStateCache/Mirrors/IMirrorStore.cs @@ -57,6 +57,17 @@ public interface IMirrorStore /// The directories. public IReadOnlyList Enumerate(); + /// + /// Lists every directory a clone is staging into, or was when it died. + /// + /// + /// A clone is written beside its mirror under a name only a clone uses, and moved into place when it + /// finishes. One whose process was killed first is never moved or removed by that clone, so it is + /// left for the maintenance sweep, which is the only thing that looks for it. + /// + /// The directories. + public IReadOnlyList EnumerateStaging(); + /// Removes a mirror and everything under it. /// The mirror directory. public void Delete(string directory); diff --git a/GitBranchStateCache/Mirrors/MirrorFetcher.cs b/GitBranchStateCache/Mirrors/MirrorFetcher.cs index 5c21c8d..eba5b69 100644 --- a/GitBranchStateCache/Mirrors/MirrorFetcher.cs +++ b/GitBranchStateCache/Mirrors/MirrorFetcher.cs @@ -134,7 +134,7 @@ private async Task CloneAsync( fileSystem.Directory.CreateDirectory(parent); - string staging = fileSystem.Path.Combine(parent, $"{MirrorStore.MirrorDirectoryName}.tmp-{Guid.NewGuid():N}"); + string staging = fileSystem.Path.Combine(parent, $"{MirrorStore.StagingPrefix}{Guid.NewGuid():N}"); metrics.RecordClone(key.Upstream); MirrorLog.Cloning(logger, key.RepositoryPath, key.Upstream); diff --git a/GitBranchStateCache/Mirrors/MirrorLog.cs b/GitBranchStateCache/Mirrors/MirrorLog.cs index 491243e..f01fb47 100644 --- a/GitBranchStateCache/Mirrors/MirrorLog.cs +++ b/GitBranchStateCache/Mirrors/MirrorLog.cs @@ -44,4 +44,16 @@ internal static partial class MirrorLog Level = LogLevel.Warning, Message = "Could not remove the idle mirror at '{Directory}'.")] public static partial void ReapFailed(ILogger logger, Exception exception, string directory); + + [LoggerMessage( + EventId = 1006, + Level = LogLevel.Information, + Message = "Removed the staging directory '{Staging}', left by a clone that did not finish.")] + public static partial void SweptStaging(ILogger logger, string staging); + + [LoggerMessage( + EventId = 1007, + Level = LogLevel.Warning, + Message = "Could not remove the staging directory '{Staging}', left by a clone that did not finish.")] + public static partial void SweepStagingFailed(ILogger logger, Exception exception, string staging); } diff --git a/GitBranchStateCache/Mirrors/MirrorMaintenanceService.cs b/GitBranchStateCache/Mirrors/MirrorMaintenanceService.cs index 297e74a..ace95c0 100644 --- a/GitBranchStateCache/Mirrors/MirrorMaintenanceService.cs +++ b/GitBranchStateCache/Mirrors/MirrorMaintenanceService.cs @@ -58,7 +58,16 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } /// - /// Measures every mirror and reaps the idle ones. + /// How much longer than the longest possible clone a staging directory must be left alone. + /// + /// + /// Covers the time a timed-out git process takes to be killed and the clone's own bookkeeping + /// either side of it, so that a clone near the end of its timeout is never swept from under it. + /// + private static readonly TimeSpan StagingMargin = TimeSpan.FromMinutes(10); + + /// + /// Removes abandoned clone staging directories, then measures every mirror and reaps the idle ones. /// internal void Sweep() { @@ -68,6 +77,8 @@ internal void Sweep() long bytes = 0; int kept = 0; + SweepStaging(settings, directories, now); + foreach (string directory in directories) { // A mirror can sit inside another when a repository path runs through a mirror.git segment @@ -92,6 +103,58 @@ internal void Sweep() ReadinessLog.ReportedMirrorSize(logger, bytes, kept); } + /// + /// Removes the staging directories of clones that can no longer be running. + /// + /// + /// A clone stages into a directory beside its mirror and discards it in a finally, which + /// never runs when the process is killed partway through: an OOM kill, an eviction or a rollout + /// during the clone of a large repository. The partial clone, possibly gigabytes, would otherwise + /// stay on the volume for good, and uncounted, since it is not a mirror. + /// + /// A clone runs for at most and then + /// configures the result within , so a staging + /// directory older than both, plus a margin, belongs to no clone still running. Age is the later of + /// creation and last write, so a filesystem that cannot report creation times errs towards keeping. + /// This runs whether or not idle mirrors are reaped, because keeping every mirror is not a reason + /// to keep what is not one. + /// + /// + private void SweepStaging(GitBranchStateCacheOptions settings, IReadOnlyList directories, DateTimeOffset now) + { + TimeSpan maxAge = settings.FetchTimeout + settings.ProbeTimeout + StagingMargin; + + foreach (string staging in mirrors.EnumerateStaging()) + { + // A mirror inside means this is a repository path segment that happens to share the name, + // not a staging directory. + if (directories.Any(directory => MirrorStore.IsInside(directory, staging))) + { + continue; + } + + try + { + IDirectoryInfo info = fileSystem.DirectoryInfo.New(staging); + DateTime touched = info.CreationTimeUtc > info.LastWriteTimeUtc + ? info.CreationTimeUtc + : info.LastWriteTimeUtc; + + if (now - new DateTimeOffset(touched, TimeSpan.Zero) <= maxAge) + { + continue; + } + + mirrors.Delete(staging); + MirrorLog.SweptStaging(logger, staging); + } + catch (Exception failure) when (failure is IOException or UnauthorizedAccessException) + { + MirrorLog.SweepStagingFailed(logger, failure, staging); + } + } + } + /// /// Reports when a mirror was last useful to anyone. /// diff --git a/GitBranchStateCache/Mirrors/MirrorStore.cs b/GitBranchStateCache/Mirrors/MirrorStore.cs index 08f4031..f0ed372 100644 --- a/GitBranchStateCache/Mirrors/MirrorStore.cs +++ b/GitBranchStateCache/Mirrors/MirrorStore.cs @@ -30,6 +30,12 @@ public sealed class MirrorStore( /// The directory name every mirror repository has. internal const string MirrorDirectoryName = "mirror.git"; + /// What a clone's staging directory is named before the suffix that makes it unique. + internal const string StagingPrefix = MirrorDirectoryName + ".tmp-"; + + /// The length of the unique suffix: a GUID written as 32 hexadecimal digits. + private const int StagingSuffixLength = 32; + private const string FetchedMarker = ".refs-fetched-at"; private const string UsedMarker = ".last-used"; @@ -111,6 +117,36 @@ public IReadOnlyList Enumerate() return [.. candidates.Where(candidate => !IsRepositoryPathSegment(candidate, candidates))]; } + /// + /// + /// Matched on the whole name a clone gives its staging directory, the prefix and a GUID's 32 + /// hexadecimal digits, rather than on the prefix alone. A repository path segment may begin + /// mirror.git.tmp- too, and that directory holds live mirrors. + /// + public IReadOnlyList EnumerateStaging() + { + string root = options.Value.MirrorRoot; + + if (!fileSystem.Directory.Exists(root)) + { + return []; + } + + return [.. fileSystem.Directory + .GetDirectories(root, StagingPrefix + "*", SearchOption.AllDirectories) + .Where(candidate => IsStagingName(fileSystem.Path.GetFileName(candidate)))]; + } + + /// + /// Reports whether a directory name is one a clone gives its staging directory. + /// + /// The directory name. + /// when it is the staging prefix followed by 32 hexadecimal digits. + internal static bool IsStagingName(string name) => + name.Length == StagingPrefix.Length + StagingSuffixLength + && name.StartsWith(StagingPrefix, StringComparison.Ordinal) + && name[StagingPrefix.Length..].All(char.IsAsciiHexDigit); + /// /// Reports whether a path lies strictly inside a directory. /// diff --git a/README.md b/README.md index ca3308a..7876a00 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,7 @@ Startup validation refuses to run with no upstreams, an upstream with an empty o Deploy **adjacent to the forge, not on-premises**. Its cost is round trips and its clients are worst served on residential links, which is the opposite placement from an object cache. `deploy/k8s` is a kustomize base: a StatefulSet with a read-write-once volume, a service, an ingress and a configmap. Start at one replica; each replica holds its own mirrors and diff cache, so replicas multiply fetch traffic and disk without improving the hit rate. -The volume is sized by the allow-list rather than by traffic, which is what makes it possible to provision in advance. Mirrors that go unqueried for longer than `MirrorIdleMaxAge` are deleted, because a deleted mirror costs one clone if it is asked for again, which is the cheapest possible way to be wrong. +The volume is sized by the allow-list rather than by traffic, which is what makes it possible to provision in advance. Mirrors that go unqueried for longer than `MirrorIdleMaxAge` are deleted, because a deleted mirror costs one clone if it is asked for again, which is the cheapest possible way to be wrong. The same sweep removes the partial clone a killed process leaves behind (`mirror.git.tmp-*`), once it is older than `FetchTimeout` plus `ProbeTimeout` plus ten minutes and so cannot belong to a clone still running. ## Instrumentation