From 86c8cf43a4b0dca0a1008cf4324b99ea0334f5d0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 07:29:38 +0000 Subject: [PATCH 1/3] Keep scanning when one file or one model response fails HashFiles let an IOException from one unreadable or dangling file escape Parallel.ForEach, and the describe loop caught only HttpRequestException, so a non-JSON response, a request timeout or a file vanishing after hashing ended the whole run with an unhandled AggregateException. HashFiles now warns about and leaves out a file it cannot read. The describe phase moves into Scan.DescribeImages, which logs and collects per-image failures, including JsonException, the timeout's OperationCanceledException and IOException, and carries on with the other images. Scan then prints a summary of what failed. Fixes #150 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0113g7mUxeizhTrH6na279Ez --- ImageDescriber.Test/ScanTests.cs | 122 +++++++++++++++++++++++++++++++ ImageDescriber/ImageHasher.cs | 17 ++++- ImageDescriber/Verbs/Scan.cs | 83 ++++++++++++++++----- 3 files changed, 203 insertions(+), 19 deletions(-) diff --git a/ImageDescriber.Test/ScanTests.cs b/ImageDescriber.Test/ScanTests.cs index f4304d3..972bf25 100644 --- a/ImageDescriber.Test/ScanTests.cs +++ b/ImageDescriber.Test/ScanTests.cs @@ -2,6 +2,12 @@ namespace ktsu.ImageDescriber.Tests; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; + using ktsu.ImageDescriber.Verbs; using ktsu.Semantics.Paths; using ktsu.Semantics.Strings; @@ -80,4 +86,120 @@ public void SanitizeFileNameHandlesSimpleName() Assert.AreEqual("sunset-over-ocean.webp", result.WeakString); } + + [TestMethod] + public void HashFilesSkipsFilesThatCannotBeRead() + { + string tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(tempDir); + + try + { + AbsoluteFilePath real = Path.Combine(tempDir, "real.jpg").As(); + AbsoluteFilePath missing = Path.Combine(tempDir, "broken.jpg").As(); + File.WriteAllBytes(real.WeakString, [0xFF, 0xD8]); + + Dictionary hashes = ImageHasher.HashFiles([real, missing]); + + Assert.AreEqual(1, hashes.Count); + Assert.IsTrue(hashes.ContainsKey(real)); + } + finally + { + Directory.Delete(tempDir, true); + } + } + + [TestMethod] + public void DescribeImagesContinuesPastPerImageFailures() + { + string tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(tempDir); + + using HttpListener listener = StartFakeOllama(out string endpoint); + try + { + AbsoluteFilePath good = Path.Combine(tempDir, "good.jpg").As(); + AbsoluteFilePath bad = Path.Combine(tempDir, "bad.jpg").As(); + AbsoluteFilePath gone = Path.Combine(tempDir, "gone.jpg").As(); + File.WriteAllBytes(good.WeakString, [0xFF, 0xD8]); + File.WriteAllBytes(bad.WeakString, [0xFF, 0xD8]); + + Dictionary> newHashPaths = new() + { + ["good"] = [good], + ["bad"] = [bad], + ["gone"] = [gone], + }; + ConcurrentBag stored = []; + + IReadOnlyList failures = Scan.DescribeImages( + newHashPaths, + endpoint.As(), + "test-model".As(), + "Describe.", + "Name it.", + maxConcurrency: 1, + stored.Add); + + Assert.AreEqual(1, stored.Count); + Assert.AreEqual("good", stored.Single().Hash); + Assert.AreEqual("a dog on a beach", stored.Single().Description); + Assert.AreEqual(2, failures.Count); + Assert.IsTrue(failures.Any(f => f.Contains("bad.jpg", StringComparison.Ordinal) && f.Contains(nameof(JsonException), StringComparison.Ordinal))); + Assert.IsTrue(failures.Any(f => f.Contains("gone.jpg", StringComparison.Ordinal))); + } + finally + { + listener.Stop(); + Directory.Delete(tempDir, true); + } + } + + /// + /// Serves /api/generate like Ollama, except that a request mentioning bad.jpg gets an HTML + /// page, as a proxy or the wrong service on the endpoint would return. + /// + private static HttpListener StartFakeOllama(out string endpoint) + { + using TcpListener portFinder = new(IPAddress.Loopback, 0); + portFinder.Start(); + int port = ((IPEndPoint)portFinder.LocalEndpoint).Port; + portFinder.Stop(); + + endpoint = $"http://localhost:{port}"; + HttpListener listener = new(); + listener.Prefixes.Add($"{endpoint}/"); + listener.Start(); + + _ = Task.Run(async () => + { + while (listener.IsListening) + { + HttpListenerContext context; + try + { + context = await listener.GetContextAsync().ConfigureAwait(false); + } + catch (HttpListenerException) + { + return; + } + catch (ObjectDisposedException) + { + return; + } + + using StreamReader reader = new(context.Request.InputStream); + string body = await reader.ReadToEndAsync().ConfigureAwait(false); + bool isBad = body.Contains("bad.jpg", StringComparison.Ordinal); + context.Response.ContentType = isBad ? "text/html" : "application/json"; + byte[] response = Encoding.UTF8.GetBytes(isBad ? "Not Ollama" : "{\"response\":\"a dog on a beach\"}"); + await context.Response.OutputStream.WriteAsync(response).ConfigureAwait(false); + context.Response.Close(); + } + }); + + return listener; + } } diff --git a/ImageDescriber/ImageHasher.cs b/ImageDescriber/ImageHasher.cs index c0779d5..74d0b5d 100644 --- a/ImageDescriber/ImageHasher.cs +++ b/ImageDescriber/ImageHasher.cs @@ -19,7 +19,22 @@ internal static Dictionary HashFiles(IReadOnlyList { - string hash = ComputeHash(filePath); + string hash; + try + { + hash = ComputeHash(filePath); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + // One unreadable, locked or dangling file must not abort hashing the rest. + lock (ConsoleLock) + { + Console.WriteLine($" Warning: could not hash {filePath}: {ex.Message}"); + } + + return; + } + results[filePath] = hash; lock (ConsoleLock) diff --git a/ImageDescriber/Verbs/Scan.cs b/ImageDescriber/Verbs/Scan.cs index 676ec3a..b717d91 100644 --- a/ImageDescriber/Verbs/Scan.cs +++ b/ImageDescriber/Verbs/Scan.cs @@ -2,8 +2,11 @@ namespace ktsu.ImageDescriber.Verbs; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; +using System.Net.Http; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -118,17 +121,65 @@ internal override void Run(Scan options) Console.WriteLine(); // Step 5: Describe new images with configurable concurrency - string descriptionPrompt = Program.Settings.DescriptionPrompt; - string fileNamePrompt = Program.Settings.SuggestedFileNamePrompt; int maxConcurrency = Math.Max(1, Program.Settings.MaxConcurrentRequests); - int current = 0; - int total = newHashPaths.Count; - Lock consoleLock = new(); Lock saveLock = new(); Console.WriteLine($"Processing with {maxConcurrency} concurrent request(s)..."); Console.WriteLine(); + IReadOnlyList failures = DescribeImages( + newHashPaths, + options.Endpoint, + options.Model, + Program.Settings.DescriptionPrompt, + Program.Settings.SuggestedFileNamePrompt, + maxConcurrency, + entry => + { + lock (saveLock) + { + Program.Settings.Descriptions[entry.Hash] = entry; + Program.Settings.Save(); + } + }); + + if (failures.Count > 0) + { + Console.WriteLine(); + Console.WriteLine($"Failed to describe {failures.Count} image(s):"); + foreach (string failure in failures) + { + Console.WriteLine($" {failure}"); + } + } + + Console.WriteLine(); + Console.WriteLine("Scan complete."); + Console.WriteLine($"Total descriptions in database: {Program.Settings.Descriptions.Count}"); + + PathString = "."; + } + + /// + /// Describes each new image and hands every finished entry to . + /// A failure on one image is logged and returned rather than thrown, so one bad file or one + /// bad or timed-out model response does not abort a run that may have been going for hours. + /// + /// One line per image that could not be described. + internal static IReadOnlyList DescribeImages( + Dictionary> newHashPaths, + OllamaEndpoint endpoint, + OllamaModelName model, + string descriptionPrompt, + string fileNamePrompt, + int maxConcurrency, + Action store) + { + int current = 0; + int total = newHashPaths.Count; + Lock consoleLock = new(); + ConcurrentQueue failures = new(); + ParallelOptions parallelOptions = new() { MaxDegreeOfParallelism = maxConcurrency }; Parallel.ForEach(newHashPaths, parallelOptions, kvp => @@ -146,10 +197,10 @@ internal override void Run(Scan options) { string pathContext = string.Join("\n", paths.Select(p => p.WeakString)); string fullPrompt = $"Known file paths for this image:\n{pathContext}\n\n{descriptionPrompt}"; - string description = OllamaClient.DescribeImageAsync(options.Endpoint, options.Model, fullPrompt, filePath).GetAwaiter().GetResult(); + string description = OllamaClient.DescribeImageAsync(endpoint, model, fullPrompt, filePath).GetAwaiter().GetResult(); string combinedFileNamePrompt = $"Image description: {description}\n\n{fileNamePrompt}"; - string rawSuggestion = OllamaClient.GenerateAsync(options.Endpoint, options.Model, combinedFileNamePrompt).GetAwaiter().GetResult(); + string rawSuggestion = OllamaClient.GenerateAsync(endpoint, model, combinedFileNamePrompt).GetAwaiter().GetResult(); FileName suggestedFileName = SanitizeFileName(rawSuggestion, filePath.FileExtension); ImageDescription entry = new() @@ -158,16 +209,12 @@ internal override void Run(Scan options) KnownPaths = [.. paths], Description = description, SuggestedFileName = suggestedFileName, - Model = options.Model, + Model = model, DescribedAt = DateTime.UtcNow, FileSizeBytes = new FileInfo(filePath.WeakString).Length, }; - lock (saveLock) - { - Program.Settings.Descriptions[hash] = entry; - Program.Settings.Save(); - } + store(entry); lock (consoleLock) { @@ -175,8 +222,11 @@ internal override void Run(Scan options) Console.WriteLine($" [{index}/{total}] Done: {description[..Math.Min(80, description.Length)]}..."); } } - catch (HttpRequestException ex) + catch (Exception ex) when (ex is HttpRequestException or JsonException or OperationCanceledException or IOException or UnauthorizedAccessException) { + // OperationCanceledException covers the TaskCanceledException HttpClient throws on timeout. + string failure = $"{filePath}: {ex.GetType().Name}: {ex.Message}"; + failures.Enqueue(failure); lock (consoleLock) { Console.WriteLine($" [{index}/{total}] Error describing {filePath.FileName}: {ex.Message}"); @@ -184,10 +234,7 @@ internal override void Run(Scan options) } }); - Console.WriteLine("Scan complete."); - Console.WriteLine($"Total descriptions in database: {Program.Settings.Descriptions.Count}"); - - PathString = "."; + return [.. failures]; } internal static FileName SanitizeFileName(string rawSuggestion, FileExtension extension) From 6967a6d28b2b7f2998d07217077f46dc9f83b7bc Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 07:36:25 +0000 Subject: [PATCH 2/3] Cover Scan's failure summary with an end-to-end test Runs the Scan verb against the fake Ollama endpoint with an image the server answers with HTML, and checks that the run completes and reports the failure. This covers the new code in Scan.Run that DescribeImages' own test cannot reach. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0113g7mUxeizhTrH6na279Ez --- ImageDescriber.Test/ScanTests.cs | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ImageDescriber.Test/ScanTests.cs b/ImageDescriber.Test/ScanTests.cs index 972bf25..05214e5 100644 --- a/ImageDescriber.Test/ScanTests.cs +++ b/ImageDescriber.Test/ScanTests.cs @@ -156,6 +156,40 @@ public void DescribeImagesContinuesPastPerImageFailures() } } + [TestMethod] + public void ScanReportsFailedImagesAndCompletes() + { + string tempDir = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + Directory.CreateDirectory(tempDir); + PersistentState originalSettings = Program.Settings; + TextWriter originalOut = Console.Out; + + using HttpListener listener = StartFakeOllama(out string endpoint); + using StringWriter output = new(); + try + { + // Every image fails, so nothing is described and nothing is saved to the real app data. + File.WriteAllBytes(Path.Combine(tempDir, "bad.jpg"), [0xFF, 0xD8]); + Program.Settings = new PersistentState(); + Console.SetOut(output); + + Scan scan = new() { PathString = tempDir, EndpointString = endpoint, ModelString = "test-model" }; + scan.Run(scan); + } + finally + { + Console.SetOut(originalOut); + Program.Settings = originalSettings; + listener.Stop(); + Directory.Delete(tempDir, true); + } + + string text = output.ToString(); + StringAssert.Contains(text, "Failed to describe 1 image(s):"); + StringAssert.Contains(text, "bad.jpg: JsonException"); + StringAssert.Contains(text, "Scan complete."); + } + /// /// Serves /api/generate like Ollama, except that a request mentioning bad.jpg gets an HTML /// page, as a proxy or the wrong service on the endpoint would return. From 9fab8e25032b6a86ab451081808695f3509d0688 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 26 Sep 2026 07:42:41 +0000 Subject: [PATCH 3/3] Move Scan's failure summary into PrintFailureSummary SonarCloud flagged Scan.Run's cognitive complexity (16, limit 15) once it printed the summary inline. Also switch the new tests to Assert.HasCount and Assert.Contains, as the MSTest analyzers suggest. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0113g7mUxeizhTrH6na279Ez --- ImageDescriber.Test/ScanTests.cs | 16 ++++++++-------- ImageDescriber/Verbs/Scan.cs | 25 ++++++++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/ImageDescriber.Test/ScanTests.cs b/ImageDescriber.Test/ScanTests.cs index 05214e5..6fae3a9 100644 --- a/ImageDescriber.Test/ScanTests.cs +++ b/ImageDescriber.Test/ScanTests.cs @@ -101,7 +101,7 @@ public void HashFilesSkipsFilesThatCannotBeRead() Dictionary hashes = ImageHasher.HashFiles([real, missing]); - Assert.AreEqual(1, hashes.Count); + Assert.HasCount(1, hashes); Assert.IsTrue(hashes.ContainsKey(real)); } finally @@ -142,12 +142,12 @@ public void DescribeImagesContinuesPastPerImageFailures() maxConcurrency: 1, stored.Add); - Assert.AreEqual(1, stored.Count); + Assert.HasCount(1, stored); Assert.AreEqual("good", stored.Single().Hash); Assert.AreEqual("a dog on a beach", stored.Single().Description); - Assert.AreEqual(2, failures.Count); - Assert.IsTrue(failures.Any(f => f.Contains("bad.jpg", StringComparison.Ordinal) && f.Contains(nameof(JsonException), StringComparison.Ordinal))); - Assert.IsTrue(failures.Any(f => f.Contains("gone.jpg", StringComparison.Ordinal))); + Assert.HasCount(2, failures); + Assert.Contains(f => f.Contains("bad.jpg", StringComparison.Ordinal) && f.Contains(nameof(JsonException), StringComparison.Ordinal), failures); + Assert.Contains(f => f.Contains("gone.jpg", StringComparison.Ordinal), failures); } finally { @@ -185,9 +185,9 @@ public void ScanReportsFailedImagesAndCompletes() } string text = output.ToString(); - StringAssert.Contains(text, "Failed to describe 1 image(s):"); - StringAssert.Contains(text, "bad.jpg: JsonException"); - StringAssert.Contains(text, "Scan complete."); + Assert.Contains("Failed to describe 1 image(s):", text); + Assert.Contains("bad.jpg: JsonException", text); + Assert.Contains("Scan complete.", text); } /// diff --git a/ImageDescriber/Verbs/Scan.cs b/ImageDescriber/Verbs/Scan.cs index b717d91..2241803 100644 --- a/ImageDescriber/Verbs/Scan.cs +++ b/ImageDescriber/Verbs/Scan.cs @@ -143,15 +143,7 @@ internal override void Run(Scan options) } }); - if (failures.Count > 0) - { - Console.WriteLine(); - Console.WriteLine($"Failed to describe {failures.Count} image(s):"); - foreach (string failure in failures) - { - Console.WriteLine($" {failure}"); - } - } + PrintFailureSummary(failures); Console.WriteLine(); Console.WriteLine("Scan complete."); @@ -160,6 +152,21 @@ internal override void Run(Scan options) PathString = "."; } + private static void PrintFailureSummary(IReadOnlyList failures) + { + if (failures.Count == 0) + { + return; + } + + Console.WriteLine(); + Console.WriteLine($"Failed to describe {failures.Count} image(s):"); + foreach (string failure in failures) + { + Console.WriteLine($" {failure}"); + } + } + /// /// Describes each new image and hands every finished entry to . /// A failure on one image is logged and returned rather than thrown, so one bad file or one