Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,11 @@ PublishScripts/
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# and except a Unity project's Packages/, which is source: Unity's package manifest and its
# resolved lock file are both meant to be committed, and a NuGet restore folder never contains
# a file by either name.
!**/[Pp]ackages/manifest.json
!**/[Pp]ackages/packages-lock.json
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
Expand Down Expand Up @@ -651,3 +656,16 @@ Temporary Items

# ImGui.ini files
imgui.ini

# Game engine projects
#
# Godot: the import cache, and the mono/temp bin+obj a C# build writes.
.godot/

# Unity: .meta files are source, not the Visual Studio C++ build artifact that the `*.meta` rule
# further up targets. Unity generates one per asset and it carries the GUID that scenes, prefabs
# and serialized references point at, so ignoring them gives every clone fresh GUIDs and silently
# breaks those references - including for a plug-in whose .dll is itself a build output. This
# negation has to come after that rule to win, and is scoped to the asset tree so the Visual
# Studio artifact stays ignored everywhere else.
!**/[Aa]ssets/**/*.meta
32 changes: 32 additions & 0 deletions FileDeduplicator.Test/FileScannerAndHasherTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -164,4 +164,36 @@ public void ParallelHashingAgreesWithSingleFileHashing()
Assert.AreEqual(FileHasher.ComputeHash(file), hashes[file]);
}
}

/// <summary>
/// One file the process cannot read must not take the whole hashing pass down with it. Hashing
/// runs under <c>Parallel.ForEach</c>, so an exception escaping the delegate surfaces as an
/// <see cref="AggregateException"/> and discards the results every other thread had already
/// produced.
/// </summary>
[TestMethod]
public void HashingSkipsAnUnreadableFileAndStillHashesTheRest()
{
// Arrange
using TempTree tree = new();
AbsoluteFilePath readable = tree.Write("readable.txt", "one");
AbsoluteFilePath alsoReadable = tree.Write("also-readable.txt", "two");

// A directory standing in for a file: File.OpenRead throws UnauthorizedAccessException for
// one on every platform and for every user. Denying permission on a real file would not --
// a privileged process, root or an elevated CI runner, reads it anyway -- so that
// arrangement would pass whether or not the exception is handled.
string deniedPath = Path.Combine(tree.Root.WeakString, "denied");
_ = Directory.CreateDirectory(deniedPath);
AbsoluteFilePath unreadable = deniedPath.As<AbsoluteFilePath>();

// Act
Dictionary<AbsoluteFilePath, string> hashes = FileHasher.HashFiles([readable, alsoReadable, unreadable]);

// Assert
Assert.HasCount(2, hashes);
Assert.AreEqual(FileHasher.ComputeHash(readable), hashes[readable]);
Assert.AreEqual(FileHasher.ComputeHash(alsoReadable), hashes[alsoReadable]);
Assert.IsFalse(hashes.ContainsKey(unreadable));
}
}
21 changes: 17 additions & 4 deletions FileDeduplicator/FileHasher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,29 @@ internal static Dictionary<AbsoluteFilePath, string> HashFiles(IReadOnlyList<Abs
}
catch (IOException ex)
{
lock (ConsoleLock)
{
Console.WriteLine($" Error hashing {filePath.FileName}: {ex.Message}");
}
ReportSkipped(filePath, ex);
}
catch (UnauthorizedAccessException ex)
{
// Not an IOException, despite reading as one: a file the process is denied access to
// would otherwise escape this delegate, and Parallel.ForEach would surface it as an
// AggregateException that discards every hash the other threads had produced. This
// matches Deduplicator.StillMatchesGroup, which catches both for the same read.
ReportSkipped(filePath, ex);
}
});

return new Dictionary<AbsoluteFilePath, string>(results);
}

private static void ReportSkipped(AbsoluteFilePath filePath, Exception ex)
{
lock (ConsoleLock)
{
Console.WriteLine($" Error hashing {filePath.FileName}: {ex.Message}");
}
}

internal static string ComputeHash(AbsoluteFilePath filePath)
{
using FileStream stream = File.OpenRead(filePath.WeakString);
Expand Down
Loading