From 85c60bedbec081628143565a2fcf346b8f1737de Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Mon, 13 Jul 2026 19:25:33 +0200 Subject: [PATCH 01/10] add add shared extension --- iosApp/VoiceInbox.xcodeproj/project.pbxproj | 169 ++++++++++++++++- iosApp/VoiceInbox/ContentView.swift | 12 ++ iosApp/VoiceInbox/IosAudioImportStore.swift | 64 +++++++ .../VoiceInbox/IosSharedImportStaging.swift | 120 ++++++++++++ iosApp/VoiceInbox/VoiceInbox.entitlements | 10 + iosApp/VoiceInboxShareExtension/Info.plist | 43 +++++ .../ShareViewController.swift | 172 ++++++++++++++++++ .../VoiceInboxShareExtension.entitlements | 10 + 8 files changed, 598 insertions(+), 2 deletions(-) create mode 100644 iosApp/VoiceInbox/IosSharedImportStaging.swift create mode 100644 iosApp/VoiceInbox/VoiceInbox.entitlements create mode 100644 iosApp/VoiceInboxShareExtension/Info.plist create mode 100644 iosApp/VoiceInboxShareExtension/ShareViewController.swift create mode 100644 iosApp/VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements diff --git a/iosApp/VoiceInbox.xcodeproj/project.pbxproj b/iosApp/VoiceInbox.xcodeproj/project.pbxproj index e9ff5f6..1a3cb12 100644 --- a/iosApp/VoiceInbox.xcodeproj/project.pbxproj +++ b/iosApp/VoiceInbox.xcodeproj/project.pbxproj @@ -19,8 +19,36 @@ 101010101010101010101036 /* IosSingleFileTranscriptionController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 101010101010101010101037 /* IosSingleFileTranscriptionController.swift */; }; 101010101010101010101039 /* IosSpeechModelStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 101010101010101010101041 /* IosSpeechModelStore.swift */; }; 101010101010101010101040 /* IosSpeechModelDirectoryPicker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 101010101010101010101042 /* IosSpeechModelDirectoryPicker.swift */; }; + 18B94FCC65FAAF5B349BF28B /* IosSharedImportStaging.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1DF805AFB19C13EC1D442253 /* IosSharedImportStaging.swift */; }; + 361EE03677F4524C001986C0 /* ShareViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = A2A88E9CFF2023E878C6EC22 /* ShareViewController.swift */; }; + 5E50C86D75D247316B5953AB /* VoiceInboxShareExtension.appex in Embed App Extensions */ = {isa = PBXBuildFile; fileRef = 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; + 9FAC76B26E6BD4116D968475 /* IosSharedImportStaging.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD030301C0E14C9932DCBFE6 /* IosSharedImportStaging.swift */; }; /* End PBXBuildFile section */ +/* Begin PBXContainerItemProxy section */ + B02BB42352D5BAE85F16E7A7 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 101010101010101010101010 /* Project object */; + proxyType = 1; + remoteGlobalIDString = D93524F1DBD24C87F8DF0EB9; + remoteInfo = VoiceInboxShareExtension; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 5885D5F7784EAD2E2CF6BB3D /* Embed App Extensions */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 2147483647; + dstPath = ""; + dstSubfolderSpec = 13; + files = ( + 5E50C86D75D247316B5953AB /* VoiceInboxShareExtension.appex in Embed App Extensions */, + ); + name = "Embed App Extensions"; + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXCopyFilesBuildPhase section */ + /* Begin PBXFileReference section */ 101010101010101010101003 /* VoiceInbox.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = VoiceInbox.app; sourceTree = BUILT_PRODUCTS_DIR; }; 101010101010101010101011 /* VoiceInboxApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VoiceInboxApp.swift; sourceTree = ""; }; @@ -35,6 +63,13 @@ 101010101010101010101037 /* IosSingleFileTranscriptionController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IosSingleFileTranscriptionController.swift; sourceTree = ""; }; 101010101010101010101041 /* IosSpeechModelStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IosSpeechModelStore.swift; sourceTree = ""; }; 101010101010101010101042 /* IosSpeechModelDirectoryPicker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IosSpeechModelDirectoryPicker.swift; sourceTree = ""; }; + 1DF805AFB19C13EC1D442253 /* IosSharedImportStaging.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = IosSharedImportStaging.swift; sourceTree = ""; }; + 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = VoiceInboxShareExtension.appex; sourceTree = BUILT_PRODUCTS_DIR; }; + A2A88E9CFF2023E878C6EC22 /* ShareViewController.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ShareViewController.swift; sourceTree = ""; }; + A3B22C5C663FBCBBFAB0A81F /* VoiceInboxShareExtension.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VoiceInboxShareExtension.entitlements; sourceTree = ""; }; + CD030301C0E14C9932DCBFE6 /* IosSharedImportStaging.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = IosSharedImportStaging.swift; path = ../VoiceInbox/IosSharedImportStaging.swift; sourceTree = ""; }; + EF9FEAB9A5BF00B4E66B996C /* VoiceInbox.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VoiceInbox.entitlements; sourceTree = ""; }; + F63411837D138D0E90F5BA1D /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -45,6 +80,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 9628A0A1DCA14071F9ADBFB8 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -53,6 +95,7 @@ children = ( 101010101010101010101013 /* VoiceInbox */, 101010101010101010101006 /* Products */, + F139417D5AA7BD6EFD2ECADD /* VoiceInboxShareExtension */, ); sourceTree = ""; }; @@ -60,6 +103,7 @@ isa = PBXGroup; children = ( 101010101010101010101003 /* VoiceInbox.app */, + 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */, ); name = Products; sourceTree = ""; @@ -79,10 +123,23 @@ 101010101010101010101026 /* IosMainScreenShellState.swift */, 101010101010101010101024 /* SettingsView.swift */, 101010101010101010101034 /* Assets.xcassets */, + 1DF805AFB19C13EC1D442253 /* IosSharedImportStaging.swift */, + EF9FEAB9A5BF00B4E66B996C /* VoiceInbox.entitlements */, ); path = VoiceInbox; sourceTree = ""; }; + F139417D5AA7BD6EFD2ECADD /* VoiceInboxShareExtension */ = { + isa = PBXGroup; + children = ( + A2A88E9CFF2023E878C6EC22 /* ShareViewController.swift */, + CD030301C0E14C9932DCBFE6 /* IosSharedImportStaging.swift */, + F63411837D138D0E90F5BA1D /* Info.plist */, + A3B22C5C663FBCBBFAB0A81F /* VoiceInboxShareExtension.entitlements */, + ); + path = VoiceInboxShareExtension; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -95,16 +152,35 @@ 101010101010101010101009 /* Sources */, 101010101010101010101035 /* Resources */, 101010101010101010101004 /* Frameworks */, + 5885D5F7784EAD2E2CF6BB3D /* Embed App Extensions */, ); buildRules = ( ); dependencies = ( + 7AAEC850980B5408864DC85C /* PBXTargetDependency */, ); name = VoiceInbox; productName = VoiceInbox; productReference = 101010101010101010101003 /* VoiceInbox.app */; productType = "com.apple.product-type.application"; }; + D93524F1DBD24C87F8DF0EB9 /* VoiceInboxShareExtension */ = { + isa = PBXNativeTarget; + buildConfigurationList = AB4D7BF990566F2ADFD3580A /* Build configuration list for PBXNativeTarget "VoiceInboxShareExtension" */; + buildPhases = ( + 76207A5A815CD15CCD17A632 /* Sources */, + 9628A0A1DCA14071F9ADBFB8 /* Frameworks */, + 84DEA1E9E8C3D8ECF1E3BF03 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = VoiceInboxShareExtension; + productName = VoiceInboxShareExtension; + productReference = 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */; + productType = "com.apple.product-type.app-extension"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -134,6 +210,7 @@ projectRoot = ""; targets = ( 101010101010101010101007 /* VoiceInbox */, + D93524F1DBD24C87F8DF0EB9 /* VoiceInboxShareExtension */, ); }; /* End PBXProject section */ @@ -147,6 +224,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 84DEA1E9E8C3D8ECF1E3BF03 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ @@ -210,11 +294,30 @@ 101010101010101010101021 /* IosScheduledTranscriptionSettingsStore.swift in Sources */, 101010101010101010101025 /* IosMainScreenShellState.swift in Sources */, 101010101010101010101022 /* SettingsView.swift in Sources */, + 18B94FCC65FAAF5B349BF28B /* IosSharedImportStaging.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + 76207A5A815CD15CCD17A632 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 361EE03677F4524C001986C0 /* ShareViewController.swift in Sources */, + 9FAC76B26E6BD4116D968475 /* IosSharedImportStaging.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + 7AAEC850980B5408864DC85C /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = VoiceInboxShareExtension; + target = D93524F1DBD24C87F8DF0EB9 /* VoiceInboxShareExtension */; + targetProxy = B02BB42352D5BAE85F16E7A7 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin XCBuildConfiguration section */ 101010101010101010101017 /* Debug */ = { isa = XCBuildConfiguration; @@ -330,8 +433,9 @@ 101010101010101010101019 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VoiceInbox/VoiceInbox.entitlements; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 2835P3MJR8; @@ -380,8 +484,9 @@ 101010101010101010101020 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; + CODE_SIGN_ENTITLEMENTS = VoiceInbox/VoiceInbox.entitlements; CODE_SIGN_STYLE = Automatic; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_TEAM = 2835P3MJR8; @@ -425,6 +530,57 @@ }; name = Release; }; + 647FF692FFBAEA14F829FBEF /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ENABLE_OBJC_WEAK = NO; + CODE_SIGN_ENTITLEMENTS = VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 2835P3MJR8; + GENERATE_INFOPLIST_FILE = NO; + INFOPLIST_FILE = VoiceInboxShareExtension/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MARKETING_VERSION = 0.1; + PRODUCT_BUNDLE_IDENTIFIER = me.maxistar.voiceinbox.ios.share; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = NO; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 912904D2DC8D23268FD953E8 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + APPLICATION_EXTENSION_API_ONLY = YES; + CLANG_ENABLE_OBJC_WEAK = NO; + CODE_SIGN_ENTITLEMENTS = VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = 2835P3MJR8; + GENERATE_INFOPLIST_FILE = NO; + INFOPLIST_FILE = VoiceInboxShareExtension/Info.plist; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + MARKETING_VERSION = 0.1; + PRODUCT_BUNDLE_IDENTIFIER = me.maxistar.voiceinbox.ios.share; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = NO; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -446,6 +602,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + AB4D7BF990566F2ADFD3580A /* Build configuration list for PBXNativeTarget "VoiceInboxShareExtension" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 912904D2DC8D23268FD953E8 /* Release */, + 647FF692FFBAEA14F829FBEF /* Debug */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 101010101010101010101010 /* Project object */; diff --git a/iosApp/VoiceInbox/ContentView.swift b/iosApp/VoiceInbox/ContentView.swift index cdc48d7..0ab12f0 100644 --- a/iosApp/VoiceInbox/ContentView.swift +++ b/iosApp/VoiceInbox/ContentView.swift @@ -4,6 +4,7 @@ import SwiftUI struct ContentView: View { private let shellState = IosMainScreenShellState() + @Environment(\.scenePhase) private var scenePhase @StateObject private var importStore = IosAudioImportStore() @StateObject private var outputStore = IosOutputDocumentStore() @StateObject private var previewPlayer = IosAudioPreviewPlayer() @@ -390,6 +391,17 @@ struct ContentView: View { .onChange(of: importStore.files) { files in previewPlayer.stopIfUnavailable(availableFileIds: Set(files.map(\.id))) } + .onAppear { + if importStore.ingestSharedImports()?.imported ?? 0 > 0 { + selectedTab = .new + } + } + .onChange(of: scenePhase) { phase in + guard phase == .active else { return } + if importStore.ingestSharedImports()?.imported ?? 0 > 0 { + selectedTab = .new + } + } .sheet(item: transcriptBinding) { transcript in NavigationStack { ScrollView { diff --git a/iosApp/VoiceInbox/IosAudioImportStore.swift b/iosApp/VoiceInbox/IosAudioImportStore.swift index 52faa0f..8dfdff3 100644 --- a/iosApp/VoiceInbox/IosAudioImportStore.swift +++ b/iosApp/VoiceInbox/IosAudioImportStore.swift @@ -166,6 +166,26 @@ final class IosAudioImportStore: ObservableObject { importMessage = summary.message } + @discardableResult + func ingestSharedImports() -> IosAudioImportSummary? { + guard let directory = try? IosSharedImportStaging.stagingDirectory(fileManager: fileManager) else { return nil } + guard let urls = try? fileManager.contentsOfDirectory( + at: directory, + includingPropertiesForKeys: [.fileSizeKey, .contentModificationDateKey], + options: [.skipsHiddenFiles] + ) else { + return nil + } + + let stagedFiles = urls.filter { $0.pathExtension.lowercased() != IosSharedImportStaging.manifestExtension } + guard !stagedFiles.isEmpty else { return nil } + + let summary = copySupportedSharedFiles(from: stagedFiles) + load() + importMessage = summary.message + return summary + } + func selectInboxFolder(_ url: URL) { let didStartAccessing = url.startAccessingSecurityScopedResource() defer { @@ -441,6 +461,50 @@ final class IosAudioImportStore: ObservableObject { return IosAudioImportSummary(imported: imported, skipped: skipped, failed: failed) } + private func copySupportedSharedFiles(from urls: [URL]) -> IosAudioImportSummary { + ensureImportDirectoryExists() + var imported = 0 + var skipped = 0 + var failed = 0 + + for stagedURL in urls { + guard isSupportedAudio(stagedURL) else { + IosSharedImportStaging.removeStagedFileAndMetadata(stagedURL, fileManager: fileManager) + skipped += 1 + continue + } + + let metadata = IosSharedImportStaging.readMetadata(for: stagedURL) + let displayName = metadata?.displayName ?? stagedURL.lastPathComponent + let importedAtMillis = metadata?.modifiedMillis ?? metadata?.stagedAtMillis ?? currentTimeMillis() + + do { + let targetFileName = nextAvailableFileName(for: displayName) + let targetURL = importDirectory.appendingPathComponent(targetFileName) + try fileManager.copyItem(at: stagedURL, to: targetURL) + _ = catalog.upsertImportedFile( + folderUri: IosAudioCatalogConstants.importedFolderUri, + documentUri: targetFileName, + displayName: displayName, + mimeType: nil, + sizeBytes: KotlinLong(longLong: fileSize(at: targetURL)), + importedAtMillis: KotlinLong(longLong: importedAtMillis), + state: AudioFileState.pending, + lastError: nil, + processedAtMillis: nil, + transcriptText: nil, + durationUs: nil + ) + IosSharedImportStaging.removeStagedFileAndMetadata(stagedURL, fileManager: fileManager) + imported += 1 + } catch { + failed += 1 + } + } + + return IosAudioImportSummary(imported: imported, skipped: skipped, failed: failed) + } + private func load() { files = catalog.importedFiles(folderUri: IosAudioCatalogConstants.importedFolderUri).map(Self.importedFile) sortFiles() diff --git a/iosApp/VoiceInbox/IosSharedImportStaging.swift b/iosApp/VoiceInbox/IosSharedImportStaging.swift new file mode 100644 index 0000000..b30fc04 --- /dev/null +++ b/iosApp/VoiceInbox/IosSharedImportStaging.swift @@ -0,0 +1,120 @@ +import Foundation +import UniformTypeIdentifiers + +enum IosSharedImportStaging { + static let appGroupIdentifier = "group.me.maxistar.voiceinbox.ios" + static let directoryName = "SharedImports" + static let manifestExtension = "json" + + static let supportedExtensions: Set = [ + "aac", "aif", "aiff", "caf", "flac", "m4a", "mp3", "mp4", "wav" + ] + + static let supportedTypeIdentifiers: [String] = [ + UTType.audio.identifier, + UTType.movie.identifier, + UTType.mpeg4Audio.identifier, + "com.microsoft.waveform-audio", + "public.mp3", + "public.aiff-audio", + "org.xiph.flac", + ] + + struct Metadata: Codable { + let stagedFileName: String + let displayName: String + let originalPathHint: String? + let sizeBytes: Int64 + let modifiedMillis: Int64? + let stagedAtMillis: Int64 + } + + static func stagingDirectory(fileManager: FileManager = .default) throws -> URL { + guard let container = fileManager.containerURL(forSecurityApplicationGroupIdentifier: appGroupIdentifier) else { + throw StagingError.appGroupUnavailable + } + let directory = container.appendingPathComponent(directoryName, isDirectory: true) + try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) + return directory + } + + static func isSupportedAudio(_ url: URL) -> Bool { + supportedExtensions.contains(url.pathExtension.lowercased()) + } + + static func uniqueStagedFileName(for displayName: String, in directory: URL, fileManager: FileManager = .default) -> String { + let sanitized = sanitizedFileName(displayName) + let base = (sanitized as NSString).deletingPathExtension + let ext = (sanitized as NSString).pathExtension + var candidate = sanitized + var suffix = 2 + + while fileManager.fileExists(atPath: directory.appendingPathComponent(candidate).path) || + fileManager.fileExists(atPath: directory.appendingPathComponent(manifestFileName(for: candidate)).path) { + candidate = ext.isEmpty ? "\(base)-\(suffix)" : "\(base)-\(suffix).\(ext)" + suffix += 1 + } + + return candidate + } + + static func manifestFileName(for stagedFileName: String) -> String { + "\(stagedFileName).\(manifestExtension)" + } + + static func writeMetadata(_ metadata: Metadata, in directory: URL, fileManager: FileManager = .default) throws { + let data = try JSONEncoder().encode(metadata) + let metadataURL = directory.appendingPathComponent(manifestFileName(for: metadata.stagedFileName)) + try data.write(to: metadataURL, options: [.atomic]) + } + + static func readMetadata(for stagedFileURL: URL) -> Metadata? { + let metadataURL = stagedFileURL + .deletingLastPathComponent() + .appendingPathComponent(manifestFileName(for: stagedFileURL.lastPathComponent)) + guard let data = try? Data(contentsOf: metadataURL) else { return nil } + return try? JSONDecoder().decode(Metadata.self, from: data) + } + + static func removeStagedFileAndMetadata(_ stagedFileURL: URL, fileManager: FileManager = .default) { + try? fileManager.removeItem(at: stagedFileURL) + let metadataURL = stagedFileURL + .deletingLastPathComponent() + .appendingPathComponent(manifestFileName(for: stagedFileURL.lastPathComponent)) + try? fileManager.removeItem(at: metadataURL) + } + + static func fileSize(at url: URL) -> Int64 { + let values = try? url.resourceValues(forKeys: [.fileSizeKey]) + return Int64(values?.fileSize ?? 0) + } + + static func fileModifiedMillis(at url: URL) -> Int64? { + let values = try? url.resourceValues(forKeys: [.contentModificationDateKey]) + return values?.contentModificationDate.map { Int64($0.timeIntervalSince1970 * 1000) } + } + + static func currentTimeMillis() -> Int64 { + Int64(Date().timeIntervalSince1970 * 1000) + } + + static func sanitizedFileName(_ name: String) -> String { + let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "._- ")) + let scalars = name.unicodeScalars.map { scalar in + allowed.contains(scalar) ? Character(scalar) : "-" + } + let sanitized = String(scalars).trimmingCharacters(in: .whitespacesAndNewlines) + return sanitized.isEmpty ? "audio-file" : sanitized + } + + enum StagingError: LocalizedError { + case appGroupUnavailable + + var errorDescription: String? { + switch self { + case .appGroupUnavailable: + "Shared import storage is not available. Check the App Group entitlement." + } + } + } +} diff --git a/iosApp/VoiceInbox/VoiceInbox.entitlements b/iosApp/VoiceInbox/VoiceInbox.entitlements new file mode 100644 index 0000000..087647e --- /dev/null +++ b/iosApp/VoiceInbox/VoiceInbox.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.me.maxistar.voiceinbox.ios + + + diff --git a/iosApp/VoiceInboxShareExtension/Info.plist b/iosApp/VoiceInboxShareExtension/Info.plist new file mode 100644 index 0000000..4beb2e5 --- /dev/null +++ b/iosApp/VoiceInboxShareExtension/Info.plist @@ -0,0 +1,43 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleDisplayName + Voice Inbox + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + $(PRODUCT_BUNDLE_PACKAGE_TYPE) + CFBundleShortVersionString + $(MARKETING_VERSION) + CFBundleVersion + $(CURRENT_PROJECT_VERSION) + NSExtension + + NSExtensionAttributes + + NSExtensionActivationRule + + NSExtensionActivationSupportsAudioWithMaxCount + 20 + NSExtensionActivationSupportsFileWithMaxCount + 20 + NSExtensionActivationSupportsMovieWithMaxCount + 20 + + + NSExtensionPointIdentifier + com.apple.share-services + NSExtensionPrincipalClass + $(PRODUCT_MODULE_NAME).ShareViewController + + + diff --git a/iosApp/VoiceInboxShareExtension/ShareViewController.swift b/iosApp/VoiceInboxShareExtension/ShareViewController.swift new file mode 100644 index 0000000..8e93de4 --- /dev/null +++ b/iosApp/VoiceInboxShareExtension/ShareViewController.swift @@ -0,0 +1,172 @@ +import UIKit +import UniformTypeIdentifiers + +final class ShareViewController: UIViewController { + private let statusLabel = UILabel() + private let doneButton = UIButton(type: .system) + + override func viewDidLoad() { + super.viewDidLoad() + configureView() + importSharedItems() + } + + private func configureView() { + view.backgroundColor = .systemBackground + + statusLabel.translatesAutoresizingMaskIntoConstraints = false + statusLabel.numberOfLines = 0 + statusLabel.textAlignment = .center + statusLabel.text = "Importing audio..." + + doneButton.translatesAutoresizingMaskIntoConstraints = false + doneButton.setTitle("Done", for: .normal) + doneButton.addTarget(self, action: #selector(closeExtension), for: .touchUpInside) + doneButton.isHidden = true + + view.addSubview(statusLabel) + view.addSubview(doneButton) + + NSLayoutConstraint.activate([ + statusLabel.leadingAnchor.constraint(equalTo: view.leadingAnchor, constant: 24), + statusLabel.trailingAnchor.constraint(equalTo: view.trailingAnchor, constant: -24), + statusLabel.centerYAnchor.constraint(equalTo: view.centerYAnchor, constant: -24), + + doneButton.topAnchor.constraint(equalTo: statusLabel.bottomAnchor, constant: 20), + doneButton.centerXAnchor.constraint(equalTo: view.centerXAnchor), + ]) + } + + private func importSharedItems() { + Task { + let result = await stageSharedItems() + await MainActor.run { + statusLabel.text = result.message + doneButton.isHidden = false + } + } + } + + private func stageSharedItems() async -> ShareImportResult { + guard let extensionItems = extensionContext?.inputItems as? [NSExtensionItem] else { + return ShareImportResult(imported: 0, skipped: 0, failed: 1) + } + + var imported = 0 + var skipped = 0 + var failed = 0 + + for item in extensionItems { + for provider in item.attachments ?? [] { + do { + if try await stageItemProvider(provider) { + imported += 1 + } else { + skipped += 1 + } + } catch { + failed += 1 + } + } + } + + return ShareImportResult(imported: imported, skipped: skipped, failed: failed) + } + + private func stageItemProvider(_ provider: NSItemProvider) async throws -> Bool { + if provider.hasItemConformingToTypeIdentifier(UTType.fileURL.identifier), + let url = try await loadURL(from: provider, typeIdentifier: UTType.fileURL.identifier) { + return try stageFile(at: url) + } + + for typeIdentifier in IosSharedImportStaging.supportedTypeIdentifiers { + if provider.hasItemConformingToTypeIdentifier(typeIdentifier), + let url = try await loadFileRepresentation(from: provider, typeIdentifier: typeIdentifier) { + return try stageFile(at: url) + } + } + + return false + } + + private func loadURL(from provider: NSItemProvider, typeIdentifier: String) async throws -> URL? { + try await withCheckedThrowingContinuation { continuation in + provider.loadItem(forTypeIdentifier: typeIdentifier, options: nil) { item, error in + if let error { + continuation.resume(throwing: error) + return + } + if let url = item as? URL { + continuation.resume(returning: url) + return + } + if let data = item as? Data, + let url = URL(dataRepresentation: data, relativeTo: nil) { + continuation.resume(returning: url) + return + } + continuation.resume(returning: nil) + } + } + } + + private func loadFileRepresentation(from provider: NSItemProvider, typeIdentifier: String) async throws -> URL? { + try await withCheckedThrowingContinuation { continuation in + provider.loadFileRepresentation(forTypeIdentifier: typeIdentifier) { url, error in + if let error { + continuation.resume(throwing: error) + return + } + continuation.resume(returning: url) + } + } + } + + private func stageFile(at sourceURL: URL) throws -> Bool { + guard IosSharedImportStaging.isSupportedAudio(sourceURL) else { return false } + + let fileManager = FileManager.default + let directory = try IosSharedImportStaging.stagingDirectory(fileManager: fileManager) + let displayName = sourceURL.lastPathComponent + let stagedFileName = IosSharedImportStaging.uniqueStagedFileName( + for: displayName, + in: directory, + fileManager: fileManager + ) + let stagedURL = directory.appendingPathComponent(stagedFileName) + + if fileManager.fileExists(atPath: stagedURL.path) { + try fileManager.removeItem(at: stagedURL) + } + try fileManager.copyItem(at: sourceURL, to: stagedURL) + + let metadata = IosSharedImportStaging.Metadata( + stagedFileName: stagedFileName, + displayName: displayName, + originalPathHint: sourceURL.lastPathComponent, + sizeBytes: IosSharedImportStaging.fileSize(at: stagedURL), + modifiedMillis: IosSharedImportStaging.fileModifiedMillis(at: sourceURL), + stagedAtMillis: IosSharedImportStaging.currentTimeMillis() + ) + try IosSharedImportStaging.writeMetadata(metadata, in: directory, fileManager: fileManager) + return true + } + + @objc private func closeExtension() { + extensionContext?.completeRequest(returningItems: nil) + } +} + +private struct ShareImportResult { + let imported: Int + let skipped: Int + let failed: Int + + var message: String { + var parts: [String] = [] + if imported > 0 { parts.append("\(imported) staged for Voice Inbox") } + if skipped > 0 { parts.append("\(skipped) skipped") } + if failed > 0 { parts.append("\(failed) failed") } + return parts.isEmpty ? "No supported audio files found." : parts.joined(separator: ", ") + } +} diff --git a/iosApp/VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements b/iosApp/VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements new file mode 100644 index 0000000..087647e --- /dev/null +++ b/iosApp/VoiceInboxShareExtension/VoiceInboxShareExtension.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.security.application-groups + + group.me.maxistar.voiceinbox.ios + + + From 7fb2a775ea02ccd34e527760f495f5a209f7ef0d Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Mon, 13 Jul 2026 20:35:51 +0200 Subject: [PATCH 02/10] add confirm dialog --- .../xcschemes/VoiceInbox.xcscheme | 79 +++++++++ .../VoiceInboxShareExtension.xcscheme | 98 +++++++++++ iosApp/VoiceInbox/ContentView.swift | 154 +++++++++++++++++- iosApp/VoiceInbox/IosAudioImportStore.swift | 45 ++++- iosApp/VoiceInbox/SettingsView.swift | 66 ++++++++ 5 files changed, 430 insertions(+), 12 deletions(-) create mode 100644 iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme create mode 100644 iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInboxShareExtension.xcscheme diff --git a/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme new file mode 100644 index 0000000..a3ade6b --- /dev/null +++ b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInboxShareExtension.xcscheme b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInboxShareExtension.xcscheme new file mode 100644 index 0000000..3924b37 --- /dev/null +++ b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInboxShareExtension.xcscheme @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/iosApp/VoiceInbox/ContentView.swift b/iosApp/VoiceInbox/ContentView.swift index 0ab12f0..35173a1 100644 --- a/iosApp/VoiceInbox/ContentView.swift +++ b/iosApp/VoiceInbox/ContentView.swift @@ -10,12 +10,16 @@ struct ContentView: View { @StateObject private var previewPlayer = IosAudioPreviewPlayer() @StateObject private var speechModelStore = IosSpeechModelStore() @StateObject private var transcriber = IosSingleFileTranscriptionController() + @StateObject private var startupPolicyStore = IosStartupProcessingPolicyStore() @State private var selectedTab = IosShellCatalogSelection.new @State private var showingImporter = false @State private var showingInboxFolderPicker = false @State private var showingOutputPicker = false @State private var showingModelImporter = false @State private var shownTranscript: String? + @State private var startupProcessingChecked = false + @State private var startupFolderRefreshChecked = false + @State private var startupProcessingPrompt: IosStartupProcessingPrompt? var body: some View { let transcriptionBackendConfigured = transcriber.backendConfigured @@ -349,6 +353,7 @@ struct ContentView: View { SettingsView( importStore: importStore, outputStore: outputStore, + startupPolicyStore: startupPolicyStore, selectInboxFolder: { showingInboxFolderPicker = true }, @@ -392,15 +397,13 @@ struct ContentView: View { previewPlayer.stopIfUnavailable(availableFileIds: Set(files.map(\.id))) } .onAppear { - if importStore.ingestSharedImports()?.imported ?? 0 > 0 { - selectedTab = .new - } + refreshStartupSources() + evaluateStartupProcessingIfNeeded() } .onChange(of: scenePhase) { phase in guard phase == .active else { return } - if importStore.ingestSharedImports()?.imported ?? 0 > 0 { - selectedTab = .new - } + refreshStartupSources() + evaluateStartupProcessingIfNeeded() } .sheet(item: transcriptBinding) { transcript in NavigationStack { @@ -419,6 +422,24 @@ struct ContentView: View { } } } + .sheet(item: $startupProcessingPrompt) { prompt in + StartupProcessingPromptView( + pendingCount: prompt.pendingCount, + onYes: { alwaysAtStartup in + startupProcessingPrompt = nil + if alwaysAtStartup { + startupPolicyStore.policy = .yes + } + startStartupProcessing() + }, + onNo: { alwaysAtStartup in + startupProcessingPrompt = nil + if alwaysAtStartup { + startupPolicyStore.policy = .no + } + } + ) + } } } @@ -438,6 +459,73 @@ struct ContentView: View { return importStore.files.first { $0.id == row.id } } + private func refreshStartupSources() { + var foundNewWork = false + if importStore.ingestSharedImports()?.imported ?? 0 > 0 { + foundNewWork = true + } + + if !startupFolderRefreshChecked, + !importStore.inboxFolderStatus.needsSelection { + startupFolderRefreshChecked = true + let pendingBeforeRefresh = importStore.pendingCount + importStore.refreshInboxFolder() + foundNewWork = foundNewWork || importStore.pendingCount > pendingBeforeRefresh + } + + if foundNewWork { + selectedTab = .new + startupProcessingChecked = false + } + } + + private func evaluateStartupProcessingIfNeeded() { + guard !startupProcessingChecked else { return } + startupProcessingChecked = true + guard importStore.pendingCount > 0 else { return } + + switch startupPolicyStore.policy { + case .ask: + startupProcessingPrompt = IosStartupProcessingPrompt(pendingCount: importStore.pendingCount) + case .yes: + startStartupProcessing() + case .no: + break + } + } + + private func startStartupProcessing() { + guard importStore.pendingCount > 0 else { return } + guard !transcriber.isActive else { + importStore.importMessage = "Found files to process, but transcription is already running." + return + } + guard transcriber.backendConfigured else { + importStore.importMessage = "Found files to process, but the iOS transcription backend is not configured." + return + } + guard speechModelStore.isReady, !speechModelStore.isBusy else { + importStore.importMessage = "Found files to process, but the speech model is not ready." + return + } + guard let outputDocument = outputStore.currentDocument() else { + outputStore.refreshAccess() + importStore.importMessage = "Found files to process, but the output file is not ready." + return + } + + previewPlayer.stop() + selectedTab = .new + transcriber.transcribeAll( + modelDirectory: speechModelStore.modelDirectory, + outputDocument: outputDocument, + store: importStore, + onFinished: { + selectedTab = .processed + } + ) + } + private func iOSModelStatusMessage( transcriptionBackendConfigured: Bool, speechModelReady: Bool @@ -469,3 +557,57 @@ private struct IosDisplayedTranscript: Identifiable { let id = UUID() let text: String } + +private struct IosStartupProcessingPrompt: Identifiable { + let id = UUID() + let pendingCount: Int +} + +private struct StartupProcessingPromptView: View { + let pendingCount: Int + let onYes: (Bool) -> Void + let onNo: (Bool) -> Void + + @State private var alwaysAtStartup = false + + var body: some View { + NavigationStack { + VStack(alignment: .leading, spacing: 20) { + VStack(alignment: .leading, spacing: 8) { + Text("Found files to process. Process now?") + .font(.title3) + .fontWeight(.semibold) + Text(summary) + .foregroundStyle(.secondary) + } + + Toggle("Always do this at startup", isOn: $alwaysAtStartup) + + Spacer() + } + .padding() + .navigationTitle("Process Files") + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button("No") { + onNo(alwaysAtStartup) + } + } + ToolbarItem(placement: .confirmationAction) { + Button("Yes") { + onYes(alwaysAtStartup) + } + .buttonStyle(.borderedProminent) + } + } + } + .presentationDetents([.medium]) + } + + private var summary: String { + if pendingCount == 1 { + return "1 file is waiting in New." + } + return "\(pendingCount) files are waiting in New." + } +} diff --git a/iosApp/VoiceInbox/IosAudioImportStore.swift b/iosApp/VoiceInbox/IosAudioImportStore.swift index 8dfdff3..e7fa3e1 100644 --- a/iosApp/VoiceInbox/IosAudioImportStore.swift +++ b/iosApp/VoiceInbox/IosAudioImportStore.swift @@ -112,6 +112,14 @@ struct IosAudioImportSummary { let imported: Int let skipped: Int let failed: Int + let importedFileIds: [Int64] + + init(imported: Int, skipped: Int, failed: Int, importedFileIds: [Int64] = []) { + self.imported = imported + self.skipped = skipped + self.failed = failed + self.importedFileIds = importedFileIds + } var message: String { var parts: [String] = [] @@ -241,6 +249,10 @@ final class IosAudioImportStore: ObservableObject { importDirectory.appendingPathComponent(documentUri) } + var pendingCount: Int { + files.count { $0.status == .pending } + } + func refresh() { load() } @@ -365,6 +377,7 @@ final class IosAudioImportStore: ObservableObject { var imported = 0 var skipped = 0 var failed = 0 + var importedFileIds: [Int64] = [] let existingByLocalName = Dictionary(uniqueKeysWithValues: files.map { ($0.localFileName, $0) }) for sourceURL in urls { @@ -394,7 +407,7 @@ final class IosAudioImportStore: ObservableObject { try fileManager.removeItem(at: targetURL) } try fileManager.copyItem(at: sourceURL, to: targetURL) - _ = catalog.upsertImportedFile( + let importedFile = catalog.upsertImportedFile( folderUri: IosAudioCatalogConstants.importedFolderUri, documentUri: targetFileName, displayName: sourceURL.lastPathComponent, @@ -407,13 +420,19 @@ final class IosAudioImportStore: ObservableObject { transcriptText: nil, durationUs: nil ) + importedFileIds.append(importedFile.id) imported += 1 } catch { failed += 1 } } - return IosAudioImportSummary(imported: imported, skipped: skipped, failed: failed) + return IosAudioImportSummary( + imported: imported, + skipped: skipped, + failed: failed, + importedFileIds: importedFileIds + ) } private func copySupportedFiles(from urls: [URL]) -> IosAudioImportSummary { @@ -421,6 +440,7 @@ final class IosAudioImportStore: ObservableObject { var imported = 0 var skipped = 0 var failed = 0 + var importedFileIds: [Int64] = [] for sourceURL in urls { guard isSupportedAudio(sourceURL) else { @@ -439,7 +459,7 @@ final class IosAudioImportStore: ObservableObject { let targetFileName = nextAvailableFileName(for: sourceURL.lastPathComponent) let targetURL = importDirectory.appendingPathComponent(targetFileName) try fileManager.copyItem(at: sourceURL, to: targetURL) - _ = catalog.upsertImportedFile( + let importedFile = catalog.upsertImportedFile( folderUri: IosAudioCatalogConstants.importedFolderUri, documentUri: targetFileName, displayName: sourceURL.lastPathComponent, @@ -452,13 +472,19 @@ final class IosAudioImportStore: ObservableObject { transcriptText: nil, durationUs: nil ) + importedFileIds.append(importedFile.id) imported += 1 } catch { failed += 1 } } - return IosAudioImportSummary(imported: imported, skipped: skipped, failed: failed) + return IosAudioImportSummary( + imported: imported, + skipped: skipped, + failed: failed, + importedFileIds: importedFileIds + ) } private func copySupportedSharedFiles(from urls: [URL]) -> IosAudioImportSummary { @@ -466,6 +492,7 @@ final class IosAudioImportStore: ObservableObject { var imported = 0 var skipped = 0 var failed = 0 + var importedFileIds: [Int64] = [] for stagedURL in urls { guard isSupportedAudio(stagedURL) else { @@ -482,7 +509,7 @@ final class IosAudioImportStore: ObservableObject { let targetFileName = nextAvailableFileName(for: displayName) let targetURL = importDirectory.appendingPathComponent(targetFileName) try fileManager.copyItem(at: stagedURL, to: targetURL) - _ = catalog.upsertImportedFile( + let importedFile = catalog.upsertImportedFile( folderUri: IosAudioCatalogConstants.importedFolderUri, documentUri: targetFileName, displayName: displayName, @@ -496,13 +523,19 @@ final class IosAudioImportStore: ObservableObject { durationUs: nil ) IosSharedImportStaging.removeStagedFileAndMetadata(stagedURL, fileManager: fileManager) + importedFileIds.append(importedFile.id) imported += 1 } catch { failed += 1 } } - return IosAudioImportSummary(imported: imported, skipped: skipped, failed: failed) + return IosAudioImportSummary( + imported: imported, + skipped: skipped, + failed: failed, + importedFileIds: importedFileIds + ) } private func load() { diff --git a/iosApp/VoiceInbox/SettingsView.swift b/iosApp/VoiceInbox/SettingsView.swift index 3cc6cfb..c348ee5 100644 --- a/iosApp/VoiceInbox/SettingsView.swift +++ b/iosApp/VoiceInbox/SettingsView.swift @@ -1,9 +1,63 @@ import Shared import SwiftUI +enum IosStartupProcessingPolicy: String, CaseIterable, Identifiable { + case ask + case yes + case no + + var id: String { rawValue } + + var title: String { + switch self { + case .ask: + "Ask" + case .yes: + "Yes" + case .no: + "No" + } + } + + var detail: String { + switch self { + case .ask: + "Ask before processing queued files when Voice Inbox starts." + case .yes: + "Automatically process queued files when Voice Inbox starts." + case .no: + "Do not process queued files automatically at startup." + } + } +} + +@MainActor +final class IosStartupProcessingPolicyStore: ObservableObject { + @Published var policy: IosStartupProcessingPolicy { + didSet { + defaults.set(policy.rawValue, forKey: Self.policyKey) + } + } + + private let defaults: UserDefaults + + init(defaults: UserDefaults = .standard) { + self.defaults = defaults + if let rawValue = defaults.string(forKey: Self.policyKey), + let storedPolicy = IosStartupProcessingPolicy(rawValue: rawValue) { + policy = storedPolicy + } else { + policy = .ask + } + } + + private static let policyKey = "iosStartupProcessingPolicy" +} + struct SettingsView: View { @ObservedObject var importStore: IosAudioImportStore @ObservedObject var outputStore: IosOutputDocumentStore + @ObservedObject var startupPolicyStore: IosStartupProcessingPolicyStore let selectInboxFolder: () -> Void let selectOutputFile: () -> Void @@ -56,6 +110,18 @@ struct SettingsView: View { } } + Section("Startup Processing") { + Picker("When queued files are found", selection: $startupPolicyStore.policy) { + ForEach(IosStartupProcessingPolicy.allCases) { policy in + Text(policy.title).tag(policy) + } + } + + Text(startupPolicyStore.policy.detail) + .font(.footnote) + .foregroundStyle(.secondary) + } + Section("About") { LabeledContent("Version", value: appVersion) Link("Website", destination: websiteURL) From 2d5ef86e1bcf1d174e133104d62b27a0d1dcf596 Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Wed, 15 Jul 2026 20:59:28 +0200 Subject: [PATCH 03/10] autostart feature --- .gitignore | 1 + README.md | 54 ++++-- .../MainActivityInstrumentedTest.kt | 68 ++++++- .../SettingsActivityInstrumentedTest.kt | 66 +++++++ .../me/maxistar/voiceinbox/MainActivity.kt | 116 +++++++++++- .../maxistar/voiceinbox/SettingsActivity.kt | 21 +++ .../StartupProcessingCoordinator.kt | 143 ++++++++++++++ .../StartupProcessingDialogFragment.kt | 71 +++++++ .../StartupProcessingPolicyStore.kt | 56 ++++++ app/src/main/res/layout/activity_settings.xml | 48 +++++ app/src/main/res/values/strings.xml | 14 ++ .../StartupProcessingCoordinatorTest.kt | 178 ++++++++++++++++++ .../StartupProcessingPolicyStoreTest.kt | 64 +++++++ .../voiceinbox/core/StartupProcessingRules.kt | 67 +++++++ .../core/StartupProcessingRulesTest.kt | 109 +++++++++++ website/src/pages/faq.astro | 12 +- website/src/pages/setup.astro | 14 +- 17 files changed, 1076 insertions(+), 26 deletions(-) create mode 100644 app/src/androidTest/java/me/maxistar/voiceinbox/SettingsActivityInstrumentedTest.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/StartupProcessingDialogFragment.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/StartupProcessingPolicyStore.kt create mode 100644 app/src/test/java/me/maxistar/voiceinbox/StartupProcessingCoordinatorTest.kt create mode 100644 app/src/test/java/me/maxistar/voiceinbox/StartupProcessingPolicyStoreTest.kt create mode 100644 shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt create mode 100644 shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt diff --git a/.gitignore b/.gitignore index 9f4d7b4..c91454d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ *.iml .gradle +/.kotlin/ /local.properties /.idea/caches /.idea/libraries diff --git a/README.md b/README.md index 565662d..1d9b103 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ into an existing plain-text (`.txt`) or Markdown (`.md`) file. 1. Download and verify the speech model on first launch. 2. Select an existing writable `.txt` or `.md` file. 3. Select a folder containing audio recordings. -4. Review discovered recordings in the **New** tab and choose **Transcribe all**. +4. Review discovered recordings in the **New** tab and choose **Transcribe all**, or + use the startup-processing policy when the app opens. 5. The application processes new recordings sequentially in foreground work and appends one entry per successful file containing its filename, recording time, and recognized text. @@ -18,6 +19,20 @@ application preserves existing text and separates each new entry with one blank line. Folder scanning runs when the application starts and when **Refresh** is selected. It scans direct children only and does not traverse nested folders. +## Automatic Processing + +Android Settings provides two independent automation options: + +- **Startup processing** runs after the app opens, restores its selections, and + finishes scanning the audio folder. It defaults to asking before processing; + it can instead transcribe automatically or leave files queued. +- **Nightly transcription** uses Android background work around the selected + time. Android may delay this work to save battery, so exact timing is not + guaranteed. + +Both options reuse the same batch transcription worker and exclude failed files, +which remain available through their individual **Retry** actions. + ## Incremental Catalog The application stores folder entries and processing state in a private SQLite @@ -83,15 +98,30 @@ Requirements: The Gradle build extracts ONNX Runtime, builds the Rust JNI library for `arm64-v8a`, and packages the required native libraries. -## iOS Shell +## iOS Application + +The repository includes a SwiftUI iOS application at +`iosApp/VoiceInbox.xcodeproj`. It uses the generated `Shared` Kotlin +Multiplatform framework for the shared catalog, transcription rules, and speech +model metadata, and links the Rust transcription engine through the native iOS +bridge. + +The current iOS application supports: + +- selecting and scanning an audio inbox folder +- importing individual audio files and receiving files through the share + extension +- selecting an existing transcript output file +- downloading or manually installing the pinned speech model +- previewing audio and transcribing or retrying individual files +- processing queued files when the application starts, with **Ask**, **Yes**, and + **No** startup policies +- viewing processed transcripts and persisting catalog state +- changing storage and startup-processing preferences from Settings -The repository includes a minimal SwiftUI iOS shell at -`iosApp/VoiceInbox.xcodeproj`. It is a KMP wiring milestone: the app launches to -a simple screen and imports the generated `Shared` Kotlin Multiplatform -framework to display speech-model metadata from shared core. The shell also -includes a Settings screen that persists scheduled transcription preferences in -iOS `UserDefaults` and normalizes them through shared scheduling rules; iOS -scheduled execution itself remains future work. +Both applications provide configurable startup processing. Unlike Android, iOS +does not currently schedule nightly background transcription. Its automatic +processing is evaluated when the application starts or becomes active. Open the project from `notes_recognition`: @@ -122,9 +152,9 @@ Xcode SDK/configuration. The lower-level shared compile checks remain: ./gradlew :shared:compileKotlinIosSimulatorArm64 :shared:compileKotlinIosX64 :shared:compileKotlinIosArm64 ``` -This shell intentionally does not implement document picking, folder scanning, -audio playback, catalog persistence, transcription execution, settings, -scheduling, output-file writing, or Rust/iOS native bridging. +The iOS and Android applications share workflow rules and catalog behavior, but +their platform integrations remain native: SwiftUI and Apple document pickers on +iOS, and Android views, the Storage Access Framework, and WorkManager on Android. ## GitHub Actions diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt index 7cbdd92..84e611c 100644 --- a/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt @@ -19,13 +19,14 @@ import androidx.test.espresso.matcher.ViewMatchers.withText import androidx.test.ext.junit.runners.AndroidJUnit4 import org.hamcrest.Matchers.containsString import org.hamcrest.Matchers.not +import org.junit.Assert.assertEquals import org.junit.Test import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) class MainActivityInstrumentedTest { @Test - fun selectionSummariesAndMenuActionsAreVisible() { + fun selectionSummariesAndCurrentMenuActionsAreVisible() { clearActivityState() ActivityScenario.launch(MainActivity::class.java).use { onView(withId(R.id.statusTitle)).check(matches(isDisplayed())) @@ -39,8 +40,7 @@ class MainActivityInstrumentedTest { openActionBarOverflowOrOptionsMenu( InstrumentationRegistry.getInstrumentation().targetContext, ) - onView(withText(R.string.menu_select_output)).check(matches(isDisplayed())) - onView(withText(R.string.menu_select_folder)).check(matches(isDisplayed())) + onView(withText(R.string.menu_settings)).check(matches(isDisplayed())) } } @@ -92,6 +92,44 @@ class MainActivityInstrumentedTest { } } + @Test + fun rememberedStartupPromptActionsPersistTheirPolicies() { + clearActivityState() + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + showStartupPrompt(scenario, pendingCount = 2) + onView(withText(R.string.startup_processing_prompt_title)).check(matches(isDisplayed())) + onView(withText(R.string.startup_processing_remember_choice)).perform(click()) + onView(withText(R.string.startup_processing_process_now)).perform(click()) + } + + val context = InstrumentationRegistry.getInstrumentation().targetContext + val store = StartupProcessingPolicyStore( + context.getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, Context.MODE_PRIVATE), + ) + assertEquals(StartupProcessingPolicy.AUTOMATIC, store.load()) + + clearActivityState() + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + showStartupPrompt(scenario, pendingCount = 1) + onView(withText(R.string.startup_processing_remember_choice)).perform(click()) + onView(withText(R.string.startup_processing_leave_queued)).perform(click()) + } + + assertEquals(StartupProcessingPolicy.LEAVE_QUEUED, store.load()) + } + + @Test + fun startupPromptSurvivesActivityRecreationWithoutASecondDecision() { + clearActivityState() + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + showStartupPrompt(scenario, pendingCount = 2) + scenario.recreate() + + onView(withText(R.string.startup_processing_prompt_title)).check(matches(isDisplayed())) + onView(withText(R.string.startup_processing_leave_queued)).perform(click()) + } + } + @Test fun failedRowsShowErrorAndRetryAction() { clearActivityState() @@ -169,9 +207,33 @@ class MainActivityInstrumentedTest { .edit() .clear() .commit() + context.getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, Context.MODE_PRIVATE) + .edit() + .clear() + .commit() context.deleteDatabase(AndroidSqlDelightAudioCatalogFactory.DATABASE_NAME) } + private fun showStartupPrompt( + scenario: ActivityScenario, + pendingCount: Int, + ) { + scenario.onActivity { activity -> + val coordinator = MainActivity::class.java.getDeclaredField("startupCoordinator") + .apply { isAccessible = true } + .get(activity) as StartupProcessingCoordinator + coordinator.beginStartupScan(99) + coordinator.setFolderReady(true) + coordinator.setOutputReady(true) + coordinator.setModelReady(true) + coordinator.setTranscriptionState(known = true, active = false) + coordinator.onStartupCatalogReady(99, pendingCount, failedCount = 0) + MainActivity::class.java.getDeclaredMethod("evaluateStartupProcessing") + .apply { isAccessible = true } + .invoke(activity) + } + } + private fun rowState(entry: AudioCatalogEntry): MainScreenRowState = MainScreenStateController.rowState( row = MainScreenRowInput( diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/SettingsActivityInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/SettingsActivityInstrumentedTest.kt new file mode 100644 index 0000000..fdd0a41 --- /dev/null +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/SettingsActivityInstrumentedTest.kt @@ -0,0 +1,66 @@ +package me.maxistar.voiceinbox + +import android.content.Context +import androidx.test.core.app.ActivityScenario +import androidx.test.espresso.Espresso.onView +import androidx.test.espresso.action.ViewActions.click +import androidx.test.espresso.action.ViewActions.scrollTo +import androidx.test.espresso.assertion.ViewAssertions.matches +import androidx.test.espresso.matcher.ViewMatchers.isChecked +import androidx.test.espresso.matcher.ViewMatchers.isDisplayed +import androidx.test.espresso.matcher.ViewMatchers.withId +import androidx.test.espresso.matcher.ViewMatchers.withText +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import me.maxistar.voiceinbox.core.StartupProcessingPolicy +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class SettingsActivityInstrumentedTest { + @Test + fun startupPolicyIsDisplayedAndPersistsWithoutChangingNightlySettings() { + val context = InstrumentationRegistry.getInstrumentation().targetContext + clearSettings(context) + val scheduledStore = ScheduledTranscriptionSettingsStore( + context.getSharedPreferences( + ScheduledTranscriptionSettingsStore.PREFERENCES_NAME, + Context.MODE_PRIVATE, + ), + ) + scheduledStore.setEnabled(true) + scheduledStore.setTime(4, 30) + + ActivityScenario.launch(SettingsActivity::class.java).use { + onView(withText(R.string.settings_storage_title)).check(matches(isDisplayed())) + onView(withText(R.string.settings_startup_processing_title)) + .perform(scrollTo()) + .check(matches(isDisplayed())) + onView(withId(R.id.startupProcessingAsk)).check(matches(isChecked())) + onView(withId(R.id.startupProcessingAutomatic)).perform(scrollTo(), click()) + onView(withId(R.id.scheduledSwitch)).perform(scrollTo()).check(matches(isChecked())) + onView(withText(R.string.settings_about_title)).perform(scrollTo()).check(matches(isDisplayed())) + } + + val startupStore = StartupProcessingPolicyStore( + context.getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, Context.MODE_PRIVATE), + ) + assertEquals(StartupProcessingPolicy.AUTOMATIC, startupStore.load()) + assertTrue(scheduledStore.load().enabled) + assertEquals(4, scheduledStore.load().hour) + assertEquals(30, scheduledStore.load().minute) + } + + private fun clearSettings(context: Context) { + context.getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, Context.MODE_PRIVATE) + .edit() + .clear() + .commit() + context.getSharedPreferences(ScheduledTranscriptionSettingsStore.PREFERENCES_NAME, Context.MODE_PRIVATE) + .edit() + .clear() + .commit() + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt index c78beab..aa53522 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt @@ -27,7 +27,7 @@ import androidx.work.WorkManager import java.util.concurrent.Executors import java.util.UUID -class MainActivity : AppCompatActivity() { +class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listener { private lateinit var statusTitle: TextView private lateinit var statusDetail: TextView private lateinit var statusProgress: ProgressBar @@ -48,6 +48,8 @@ class MainActivity : AppCompatActivity() { private lateinit var folderScanner: AudioFolderScanner private lateinit var catalog: SqlDelightAudioCatalogRepository private lateinit var modelReadiness: SpeechModelReadinessManager + private lateinit var startupPolicyStore: StartupProcessingPolicyStore + private lateinit var startupCoordinator: StartupProcessingCoordinator private val folderExecutor = Executors.newSingleThreadExecutor() private var modelReady = false @@ -82,6 +84,7 @@ class MainActivity : AppCompatActivity() { private var previewEntryId: Long? = null private var previewState = PreviewPlaybackState.IDLE private var activityDestroyed = false + private var scanGeneration = 0L private val outputPicker = registerForActivityResult( ActivityResultContracts.OpenDocument(), @@ -114,6 +117,12 @@ class MainActivity : AppCompatActivity() { folderScanner = AudioFolderScanner(contentResolver) catalog = AndroidSqlDelightAudioCatalogFactory(this).create() modelReadiness = getSharedModelReadiness(SpeechModelRepository(noBackupFilesDir.resolve("models"))) + startupPolicyStore = StartupProcessingPolicyStore( + getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, MODE_PRIVATE), + ) + startupCoordinator = StartupProcessingCoordinator.restore( + savedInstanceState?.getString(STATE_STARTUP_PROCESSING_STAGE), + ) ScheduledTranscriptionScheduler.sync( this, ScheduledTranscriptionSettingsStore( @@ -140,6 +149,11 @@ class MainActivity : AppCompatActivity() { refreshModel() } + override fun onSaveInstanceState(outState: Bundle) { + outState.putString(STATE_STARTUP_PROCESSING_STAGE, startupCoordinator.savedStage()) + super.onSaveInstanceState(outState) + } + override fun onCreateOptionsMenu(menu: Menu): Boolean { menuInflater.inflate(R.menu.main_options, menu) updateMenu(menu) @@ -200,11 +214,18 @@ class MainActivity : AppCompatActivity() { runOnUiThread { if (activityDestroyed) return@runOnUiThread outputUri = stored + startupCoordinator.setOutputReady(true) updateOutputSummary(metadata.displayName) updateControls() + evaluateStartupProcessing() } }.onFailure { selectionStore.clearOutputUri() + runOnUiThread { + if (activityDestroyed) return@runOnUiThread + startupCoordinator.setOutputReady(false) + evaluateStartupProcessing() + } } } } @@ -224,16 +245,19 @@ class MainActivity : AppCompatActivity() { if (activityDestroyed) return@runOnUiThread folderChecking = false folderUri = stored + startupCoordinator.setFolderReady(true) updateFolderSummary(name) updateControls() refreshCatalog() - scanFolder() + scanFolder(FolderScanOrigin.STARTUP) } }.onFailure { selectionStore.clearFolderUri() runOnUiThread { if (activityDestroyed) return@runOnUiThread folderChecking = false + startupCoordinator.setFolderReady(false) + evaluateStartupProcessing() } showError(it.message ?: "Audio folder is not readable") } @@ -257,12 +281,21 @@ class MainActivity : AppCompatActivity() { runOnUiThread { if (activityDestroyed) return@runOnUiThread outputUri = uri + startupCoordinator.setOutputReady(true) updateOutputSummary(metadata.displayName) statusError = null renderStatusBlock() updateControls() + evaluateStartupProcessing() + } + }.onFailure { + runOnUiThread { + if (activityDestroyed) return@runOnUiThread + startupCoordinator.setOutputReady(false) + evaluateStartupProcessing() } - }.onFailure { showError(it.message ?: "Output file is not writable") } + showError(it.message ?: "Output file is not writable") + } } } @@ -286,6 +319,7 @@ class MainActivity : AppCompatActivity() { if (activityDestroyed) return@runOnUiThread folderChecking = false folderUri = uri + startupCoordinator.setFolderReady(true) updateFolderSummary(name) statusError = null renderStatusBlock() @@ -297,15 +331,21 @@ class MainActivity : AppCompatActivity() { runOnUiThread { if (activityDestroyed) return@runOnUiThread folderChecking = false + startupCoordinator.setFolderReady(false) + evaluateStartupProcessing() } showError(it.message ?: "Audio folder is not readable") } } } - private fun scanFolder() { + private fun scanFolder(origin: FolderScanOrigin = FolderScanOrigin.USER) { val folder = folderUri ?: return if (folderChecking || folderScanQueued || scanning || transcriptionActive()) return + val generation = ++scanGeneration + val startupGeneration = generation.takeIf { + origin == FolderScanOrigin.STARTUP && startupCoordinator.beginStartupScan(generation) + } folderScanQueued = true transcriptionFinished = false transcriptionPhase = null @@ -338,20 +378,22 @@ class MainActivity : AppCompatActivity() { folderScanQueued = false scanning = false scanMessage = "Scan complete: $count audio files" - refreshCatalog() + refreshCatalog(startupGeneration) } }.onFailure { error -> runOnUiThread { if (activityDestroyed) return@runOnUiThread folderScanQueued = false scanning = false + startupGeneration?.let(startupCoordinator::onStartupScanFailed) + evaluateStartupProcessing() showError(error.message ?: "Folder scan failed") } } } } - private fun refreshCatalog() { + private fun refreshCatalog(startupGeneration: Long? = null) { if (activityDestroyed) return val folder = folderUri?.toString() if (folder == null) { @@ -371,9 +413,20 @@ class MainActivity : AppCompatActivity() { runOnUiThread { if (activityDestroyed) return@runOnUiThread pendingCount = newEntries.count { it.state == AudioFileState.PENDING } + val catalogFailedCount = newEntries.count { it.state == AudioFileState.FAILED } renderEntries(entries) renderStatusBlock() updateControls() + if (startupGeneration != null) { + startupCoordinator.onStartupCatalogReady( + startupGeneration, + pendingCount, + catalogFailedCount, + ) + } else { + startupCoordinator.onCatalogRefreshed(pendingCount, catalogFailedCount) + } + evaluateStartupProcessing() } } } @@ -566,6 +619,8 @@ class MainActivity : AppCompatActivity() { setModelUi(state.message, loading = false, canDownload = true) } } + startupCoordinator.setModelReady(modelReady) + evaluateStartupProcessing() } private fun setModelUi(message: String, loading: Boolean, canDownload: Boolean) { @@ -587,6 +642,7 @@ class MainActivity : AppCompatActivity() { WorkInfo.State.RUNNING, -> { modelReady = false + startupCoordinator.setModelReady(false) val bytes = info.progress.getLong(SpeechModelDownloadWorker.KEY_BYTES_DOWNLOADED, 0) val total = info.progress.getLong( SpeechModelDownloadWorker.KEY_TOTAL_BYTES, @@ -600,6 +656,7 @@ class MainActivity : AppCompatActivity() { modelDownloadProgress = ((bytes * 100) / total.coerceAtLeast(1)).toInt() renderStatusBlock() updateControls() + evaluateStartupProcessing() } WorkInfo.State.SUCCEEDED -> { if (shouldHandleModelInstallSuccess(info.id.toString())) { @@ -609,6 +666,7 @@ class MainActivity : AppCompatActivity() { } WorkInfo.State.FAILED -> { modelReady = false + startupCoordinator.setModelReady(false) setModelUi( info.outputData.getString(SpeechModelDownloadWorker.KEY_ERROR) ?: "Speech model download failed", @@ -616,6 +674,7 @@ class MainActivity : AppCompatActivity() { true, ) updateControls() + evaluateStartupProcessing() } else -> Unit } @@ -627,6 +686,11 @@ class MainActivity : AppCompatActivity() { .getWorkInfosForUniqueWorkLiveData(TranscriptionWorker.UNIQUE_WORK_NAME) .observe(this) { infos -> transcriptionState = classifyTranscriptionState(infos) + startupCoordinator.setTranscriptionState( + known = true, + active = transcriptionActive(), + ) + evaluateStartupProcessing() if (infos.isEmpty()) { transcriptionFinished = false renderStatusBlock() @@ -711,6 +775,40 @@ class MainActivity : AppCompatActivity() { currentSessionTranscriptionWorkId = TranscriptionWorker.enqueueAll(this, folder, output) } + private fun evaluateStartupProcessing() { + when (val decision = startupCoordinator.evaluate(startupPolicyStore.load())) { + is StartupProcessingDecision.Prompt -> { + if ( + !supportFragmentManager.isStateSaved && + supportFragmentManager.findFragmentByTag(StartupProcessingDialogFragment.TAG) == null + ) { + StartupProcessingDialogFragment.newInstance(decision.pendingCount) + .show(supportFragmentManager, StartupProcessingDialogFragment.TAG) + } + } + is StartupProcessingDecision.Start -> startBatchTranscription() + is StartupProcessingDecision.Finish, + StartupProcessingDecision.Wait, + -> Unit + } + } + + override fun onStartupProcessingConfirmed(remember: Boolean) { + if (!startupCoordinator.confirmPrompt()) return + if (remember) { + startupPolicyStore.save(StartupProcessingPolicy.AUTOMATIC) + } + evaluateStartupProcessing() + } + + override fun onStartupProcessingDeclined(remember: Boolean) { + if (!startupCoordinator.declinePrompt()) return + if (remember) { + startupPolicyStore.save(StartupProcessingPolicy.LEAVE_QUEUED) + } + updateControls() + } + private fun transcriptionActive(): Boolean = transcriptionState == TranscriptionObservationState.ACTIVE private fun classifyTranscriptionState(infos: List): TranscriptionObservationState { @@ -942,10 +1040,16 @@ class MainActivity : AppCompatActivity() { private data class PreviewButtonTag(val entryId: Long) private data class RetryButtonTag(val entryId: Long) + private enum class FolderScanOrigin { + STARTUP, + USER, + } + private companion object { const val MENU_ENTRY_PLAY = 1 const val MENU_ENTRY_RETRY = 2 const val MENU_ENTRY_SHOW_TEXT = 3 + const val STATE_STARTUP_PROCESSING_STAGE = "startup-processing-stage" val ACTIVE_WORK_STATES = setOf( WorkInfo.State.ENQUEUED, WorkInfo.State.BLOCKED, diff --git a/app/src/main/java/me/maxistar/voiceinbox/SettingsActivity.kt b/app/src/main/java/me/maxistar/voiceinbox/SettingsActivity.kt index 26623d0..eb02b7a 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SettingsActivity.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SettingsActivity.kt @@ -9,6 +9,7 @@ import android.net.Uri import android.os.Bundle import android.view.MenuItem import android.view.View +import android.widget.RadioGroup import android.widget.TextView import android.widget.Toast import androidx.activity.enableEdgeToEdge @@ -21,6 +22,7 @@ import java.util.concurrent.Executors class SettingsActivity : AppCompatActivity() { private lateinit var settingsStore: ScheduledTranscriptionSettingsStore + private lateinit var startupPolicyStore: StartupProcessingPolicyStore private lateinit var selectionStore: DocumentSelectionStore private lateinit var documentAccess: DocumentAccess private lateinit var folderScanner: AudioFolderScanner @@ -63,6 +65,9 @@ class SettingsActivity : AppCompatActivity() { settingsStore = ScheduledTranscriptionSettingsStore( getSharedPreferences(ScheduledTranscriptionSettingsStore.PREFERENCES_NAME, MODE_PRIVATE), ) + startupPolicyStore = StartupProcessingPolicyStore( + getSharedPreferences(StartupProcessingPolicyStore.PREFERENCES_NAME, MODE_PRIVATE), + ) selectionStore = DocumentSelectionStore( getSharedPreferences(DocumentSelectionStore.PREFERENCES_NAME, MODE_PRIVATE), ) @@ -90,6 +95,22 @@ class SettingsActivity : AppCompatActivity() { findViewById(R.id.settingsLegalRow).setOnClickListener { openExternalUrl(LEGAL_URL) } + val startupPolicyGroup = findViewById(R.id.startupProcessingPolicy) + startupPolicyGroup.check( + when (startupPolicyStore.load()) { + StartupProcessingPolicy.ASK -> R.id.startupProcessingAsk + StartupProcessingPolicy.AUTOMATIC -> R.id.startupProcessingAutomatic + StartupProcessingPolicy.LEAVE_QUEUED -> R.id.startupProcessingLeaveQueued + }, + ) + startupPolicyGroup.setOnCheckedChangeListener { _, checkedId -> + val policy = when (checkedId) { + R.id.startupProcessingAutomatic -> StartupProcessingPolicy.AUTOMATIC + R.id.startupProcessingLeaveQueued -> StartupProcessingPolicy.LEAVE_QUEUED + else -> StartupProcessingPolicy.ASK + } + startupPolicyStore.save(policy) + } scheduledSwitch.isChecked = settings.enabled scheduledSwitch.setOnCheckedChangeListener { _, enabled -> settings = settingsStore.setEnabled(enabled) diff --git a/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt new file mode 100644 index 0000000..a2adfa6 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt @@ -0,0 +1,143 @@ +package me.maxistar.voiceinbox + +import me.maxistar.voiceinbox.core.StartupProcessingDecision +import me.maxistar.voiceinbox.core.StartupProcessingInput +import me.maxistar.voiceinbox.core.StartupProcessingPolicy +import me.maxistar.voiceinbox.core.StartupProcessingRules + +class StartupProcessingCoordinator private constructor( + private var stage: Stage, +) { + private var startupScanGeneration: Long? = null + private var startupScanComplete = false + private var startupScanFailed = false + private var catalogReady = false + private var pendingCount = 0 + private var failedCount = 0 + private var folderReady = false + private var outputReady = false + private var modelReady = false + private var transcriptionStateKnown = false + private var transcriptionActive = false + + constructor() : this(Stage.ACTIVE) + + fun beginStartupScan(generation: Long): Boolean { + if (stage == Stage.COMPLETE) return false + startupScanGeneration = generation + startupScanComplete = false + startupScanFailed = false + catalogReady = false + pendingCount = 0 + failedCount = 0 + return true + } + + fun onStartupCatalogReady( + generation: Long, + pendingCount: Int, + failedCount: Int, + ) { + if (generation != startupScanGeneration || stage == Stage.COMPLETE) return + startupScanComplete = true + startupScanFailed = false + catalogReady = true + this.pendingCount = pendingCount.coerceAtLeast(0) + this.failedCount = failedCount.coerceAtLeast(0) + } + + fun onCatalogRefreshed(pendingCount: Int, failedCount: Int) { + if (!startupScanComplete || stage == Stage.COMPLETE) return + catalogReady = true + this.pendingCount = pendingCount.coerceAtLeast(0) + this.failedCount = failedCount.coerceAtLeast(0) + } + + fun onStartupScanFailed(generation: Long) { + if (generation != startupScanGeneration || stage == Stage.COMPLETE) return + startupScanComplete = false + startupScanFailed = true + catalogReady = false + } + + fun setFolderReady(ready: Boolean) { + folderReady = ready + } + + fun setOutputReady(ready: Boolean) { + outputReady = ready + } + + fun setModelReady(ready: Boolean) { + modelReady = ready + } + + fun setTranscriptionState(known: Boolean, active: Boolean) { + if (transcriptionStateKnown && transcriptionActive && known && !active) { + catalogReady = false + } + transcriptionStateKnown = known + transcriptionActive = active + } + + fun evaluate(policy: StartupProcessingPolicy): StartupProcessingDecision { + if (stage == Stage.COMPLETE || stage == Stage.PROMPTING) { + return StartupProcessingDecision.Wait + } + val decision = StartupProcessingRules.decide( + StartupProcessingInput( + policy = if (stage == Stage.CONFIRMED) StartupProcessingPolicy.AUTOMATIC else policy, + startupScanComplete = startupScanComplete, + startupScanFailed = startupScanFailed, + catalogReady = catalogReady, + pendingCount = pendingCount, + failedCount = failedCount, + folderReady = folderReady, + outputReady = outputReady, + modelReady = modelReady, + transcriptionStateKnown = transcriptionStateKnown, + transcriptionActive = transcriptionActive, + ), + ) + when (decision) { + is StartupProcessingDecision.Prompt -> stage = Stage.PROMPTING + is StartupProcessingDecision.Start, + is StartupProcessingDecision.Finish, + -> stage = Stage.COMPLETE + StartupProcessingDecision.Wait -> Unit + } + return decision + } + + fun confirmPrompt(): Boolean { + if (stage != Stage.PROMPTING) return false + stage = Stage.CONFIRMED + return true + } + + fun declinePrompt(): Boolean { + if (stage != Stage.PROMPTING) return false + stage = Stage.COMPLETE + return true + } + + fun savedStage(): String = stage.name + + fun isComplete(): Boolean = stage == Stage.COMPLETE + + companion object { + fun restore(savedStage: String?): StartupProcessingCoordinator = + StartupProcessingCoordinator( + savedStage + ?.let { stored -> Stage.entries.firstOrNull { it.name == stored } } + ?: Stage.ACTIVE, + ) + } + + private enum class Stage { + ACTIVE, + PROMPTING, + CONFIRMED, + COMPLETE, + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingDialogFragment.kt b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingDialogFragment.kt new file mode 100644 index 0000000..be337b1 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingDialogFragment.kt @@ -0,0 +1,71 @@ +package me.maxistar.voiceinbox + +import android.app.Dialog +import android.content.Context +import android.content.DialogInterface +import android.os.Bundle +import android.widget.CheckBox +import androidx.appcompat.app.AlertDialog +import androidx.appcompat.app.AppCompatDialogFragment + +class StartupProcessingDialogFragment : AppCompatDialogFragment() { + private lateinit var listener: Listener + private var callbackDelivered = false + + override fun onAttach(context: Context) { + super.onAttach(context) + listener = context as? Listener + ?: error("Host Activity must implement StartupProcessingDialogFragment.Listener") + } + + override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { + val pendingCount = requireArguments().getInt(ARG_PENDING_COUNT) + val rememberChoice = CheckBox(requireContext()).apply { + text = getString(R.string.startup_processing_remember_choice) + } + return AlertDialog.Builder(requireContext()) + .setTitle(R.string.startup_processing_prompt_title) + .setMessage( + resources.getQuantityString( + R.plurals.startup_processing_prompt_message, + pendingCount, + pendingCount, + ), + ) + .setView(rememberChoice) + .setPositiveButton(R.string.startup_processing_process_now) { _, _ -> + callbackDelivered = true + listener.onStartupProcessingConfirmed(rememberChoice.isChecked) + } + .setNegativeButton(R.string.startup_processing_leave_queued) { _, _ -> + callbackDelivered = true + listener.onStartupProcessingDeclined(rememberChoice.isChecked) + } + .create() + } + + override fun onCancel(dialog: DialogInterface) { + super.onCancel(dialog) + if (!callbackDelivered) { + callbackDelivered = true + listener.onStartupProcessingDeclined(remember = false) + } + } + + interface Listener { + fun onStartupProcessingConfirmed(remember: Boolean) + fun onStartupProcessingDeclined(remember: Boolean) + } + + companion object { + const val TAG = "startup-processing-prompt" + private const val ARG_PENDING_COUNT = "pending-count" + + fun newInstance(pendingCount: Int): StartupProcessingDialogFragment = + StartupProcessingDialogFragment().apply { + arguments = Bundle().apply { + putInt(ARG_PENDING_COUNT, pendingCount) + } + } + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingPolicyStore.kt b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingPolicyStore.kt new file mode 100644 index 0000000..159bc51 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingPolicyStore.kt @@ -0,0 +1,56 @@ +package me.maxistar.voiceinbox + +import android.content.SharedPreferences +import me.maxistar.voiceinbox.core.StartupProcessingPolicy + +interface StartupProcessingPolicyStorage { + fun loadRaw(): String? + fun saveRaw(value: String) +} + +class StartupProcessingPolicyStore( + private val storage: StartupProcessingPolicyStorage, +) { + constructor(preferences: SharedPreferences) : this( + SharedPreferencesStartupProcessingPolicyStorage(preferences), + ) + + fun load(): StartupProcessingPolicy = + when (storage.loadRaw()) { + VALUE_AUTOMATIC -> StartupProcessingPolicy.AUTOMATIC + VALUE_LEAVE_QUEUED -> StartupProcessingPolicy.LEAVE_QUEUED + VALUE_ASK -> StartupProcessingPolicy.ASK + else -> StartupProcessingPolicy.ASK + } + + fun save(policy: StartupProcessingPolicy) { + storage.saveRaw( + when (policy) { + StartupProcessingPolicy.ASK -> VALUE_ASK + StartupProcessingPolicy.AUTOMATIC -> VALUE_AUTOMATIC + StartupProcessingPolicy.LEAVE_QUEUED -> VALUE_LEAVE_QUEUED + }, + ) + } + + companion object { + const val PREFERENCES_NAME = "startup_processing" + private const val VALUE_ASK = "ask" + private const val VALUE_AUTOMATIC = "automatic" + private const val VALUE_LEAVE_QUEUED = "leave_queued" + } +} + +private class SharedPreferencesStartupProcessingPolicyStorage( + private val preferences: SharedPreferences, +) : StartupProcessingPolicyStorage { + override fun loadRaw(): String? = preferences.getString(KEY_POLICY, null) + + override fun saveRaw(value: String) { + preferences.edit().putString(KEY_POLICY, value).apply() + } + + companion object { + private const val KEY_POLICY = "startup_processing_policy" + } +} diff --git a/app/src/main/res/layout/activity_settings.xml b/app/src/main/res/layout/activity_settings.xml index f9391d7..aae5355 100644 --- a/app/src/main/res/layout/activity_settings.xml +++ b/app/src/main/res/layout/activity_settings.xml @@ -80,6 +80,54 @@ android:text="@string/output_not_selected" /> + + + + + + + + + + + + + + + Scheduled time: %1$02d:%2$02d Change scheduled time Android may delay background work to save battery. The feature is off by default. + Startup processing + Choose what happens when Voice Inbox opens and finds queued notes. + Ask every time + Transcribe automatically + Leave files queued + Startup processing runs only when you open the app. Nightly transcription is a separate background option. + Process queued notes? + + %1$d file is ready to transcribe. + %1$d files are ready to transcribe. + + Always do this when Voice Inbox starts + Process now + Leave queued Output: not selected Output: %1$s Select output file diff --git a/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingCoordinatorTest.kt b/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingCoordinatorTest.kt new file mode 100644 index 0000000..e8c42fd --- /dev/null +++ b/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingCoordinatorTest.kt @@ -0,0 +1,178 @@ +package me.maxistar.voiceinbox + +import me.maxistar.voiceinbox.core.StartupProcessingDecision +import me.maxistar.voiceinbox.core.StartupProcessingPolicy +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class StartupProcessingCoordinatorTest { + @Test + fun callbackOrderingWaitsUntilEveryInputIsReady() { + val coordinator = StartupProcessingCoordinator() + coordinator.setFolderReady(true) + coordinator.setOutputReady(true) + coordinator.setModelReady(true) + coordinator.setTranscriptionState(known = true, active = false) + + assertEquals(StartupProcessingDecision.Wait, coordinator.evaluate(StartupProcessingPolicy.ASK)) + assertTrue(coordinator.beginStartupScan(1)) + assertEquals(StartupProcessingDecision.Wait, coordinator.evaluate(StartupProcessingPolicy.ASK)) + coordinator.onStartupCatalogReady(1, pendingCount = 2, failedCount = 0) + + assertEquals( + StartupProcessingDecision.Prompt(2), + coordinator.evaluate(StartupProcessingPolicy.ASK), + ) + } + + @Test + fun staleStartupGenerationIsIgnored() { + val coordinator = readyCoordinator(generation = 2, completeCatalog = false) + coordinator.onStartupCatalogReady(1, pendingCount = 3, failedCount = 0) + + assertEquals( + StartupProcessingDecision.Wait, + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + } + + @Test + fun delayedPrerequisiteReevaluatesToAutomaticStart() { + val coordinator = readyCoordinator() + coordinator.setOutputReady(false) + assertEquals( + StartupProcessingDecision.Wait, + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + + coordinator.setOutputReady(true) + assertEquals( + StartupProcessingDecision.Start(1), + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + assertTrue(coordinator.isComplete()) + } + + @Test + fun failedStartupScanCompletesEvaluation() { + val coordinator = StartupProcessingCoordinator() + assertTrue(coordinator.beginStartupScan(4)) + coordinator.onStartupScanFailed(4) + + coordinator.evaluate(StartupProcessingPolicy.ASK) + + assertTrue(coordinator.isComplete()) + } + + @Test + fun restoredPromptDoesNotEmitAnotherPrompt() { + val original = readyCoordinator() + assertEquals( + StartupProcessingDecision.Prompt(1), + original.evaluate(StartupProcessingPolicy.ASK), + ) + + val restored = StartupProcessingCoordinator.restore(original.savedStage()) + + assertEquals(StartupProcessingDecision.Wait, restored.evaluate(StartupProcessingPolicy.ASK)) + } + + @Test + fun confirmedPromptWaitsForRecreatedReadinessThenStarts() { + val original = readyCoordinator() + original.evaluate(StartupProcessingPolicy.ASK) + val restored = StartupProcessingCoordinator.restore(original.savedStage()) + assertTrue(restored.beginStartupScan(7)) + assertTrue(restored.confirmPrompt()) + assertEquals(StartupProcessingDecision.Wait, restored.evaluate(StartupProcessingPolicy.ASK)) + + prepareReady(restored, generation = 7) + + assertEquals( + StartupProcessingDecision.Start(1), + restored.evaluate(StartupProcessingPolicy.ASK), + ) + } + + @Test + fun declinedPromptIsTerminalAndManualRefreshCannotReopenIt() { + val coordinator = readyCoordinator() + coordinator.evaluate(StartupProcessingPolicy.ASK) + assertTrue(coordinator.declinePrompt()) + + coordinator.onCatalogRefreshed(pendingCount = 5, failedCount = 0) + + assertEquals(StartupProcessingDecision.Wait, coordinator.evaluate(StartupProcessingPolicy.ASK)) + assertFalse(coordinator.beginStartupScan(8)) + } + + @Test + fun aFreshCoordinatorEvaluatesANewLaunch() { + val previous = readyCoordinator() + previous.evaluate(StartupProcessingPolicy.LEAVE_QUEUED) + assertTrue(previous.isComplete()) + + val nextLaunch = readyCoordinator() + + assertEquals( + StartupProcessingDecision.Start(1), + nextLaunch.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + } + + @Test + fun activeWorkSuppressesHandoffUntilCatalogRefreshes() { + val coordinator = readyCoordinator() + coordinator.setTranscriptionState(known = true, active = true) + assertEquals( + StartupProcessingDecision.Wait, + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + + coordinator.setTranscriptionState(known = true, active = false) + assertEquals( + StartupProcessingDecision.Wait, + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC), + ) + coordinator.onCatalogRefreshed(pendingCount = 0, failedCount = 0) + + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC) + assertTrue(coordinator.isComplete()) + } + + @Test + fun failedOnlyCatalogDoesNotStart() { + val coordinator = readyCoordinator(pendingCount = 0, failedCount = 3) + + coordinator.evaluate(StartupProcessingPolicy.AUTOMATIC) + + assertTrue(coordinator.isComplete()) + } + + private fun readyCoordinator( + generation: Long = 1, + pendingCount: Int = 1, + failedCount: Int = 0, + completeCatalog: Boolean = true, + ): StartupProcessingCoordinator = + StartupProcessingCoordinator().also { coordinator -> + coordinator.beginStartupScan(generation) + coordinator.setFolderReady(true) + coordinator.setOutputReady(true) + coordinator.setModelReady(true) + coordinator.setTranscriptionState(known = true, active = false) + if (completeCatalog) { + coordinator.onStartupCatalogReady(generation, pendingCount, failedCount) + } + } + + private fun prepareReady(coordinator: StartupProcessingCoordinator, generation: Long) { + coordinator.setFolderReady(true) + coordinator.setOutputReady(true) + coordinator.setModelReady(true) + coordinator.setTranscriptionState(known = true, active = false) + coordinator.onStartupCatalogReady(generation, pendingCount = 1, failedCount = 0) + } +} diff --git a/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingPolicyStoreTest.kt b/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingPolicyStoreTest.kt new file mode 100644 index 0000000..be1c0f0 --- /dev/null +++ b/app/src/test/java/me/maxistar/voiceinbox/StartupProcessingPolicyStoreTest.kt @@ -0,0 +1,64 @@ +package me.maxistar.voiceinbox + +import me.maxistar.voiceinbox.core.ScheduledTranscriptionSettings +import me.maxistar.voiceinbox.core.ScheduledTranscriptionSettingsStorage +import me.maxistar.voiceinbox.core.StartupProcessingPolicy +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class StartupProcessingPolicyStoreTest { + @Test + fun missingAndUnknownValuesDefaultToAsk() { + val storage = FakeStartupStorage() + val store = StartupProcessingPolicyStore(storage) + + assertEquals(StartupProcessingPolicy.ASK, store.load()) + storage.raw = "future-value" + assertEquals(StartupProcessingPolicy.ASK, store.load()) + } + + @Test + fun policiesRoundTripThroughStableValues() { + val storage = FakeStartupStorage() + val store = StartupProcessingPolicyStore(storage) + + StartupProcessingPolicy.entries.forEach { policy -> + store.save(policy) + assertEquals(policy, store.load()) + } + } + + @Test + fun startupPolicyDoesNotChangeScheduledSettings() { + val scheduledStorage = FakeScheduledStorage( + ScheduledTranscriptionSettings(enabled = true, hour = 4, minute = 30), + ) + val scheduledStore = ScheduledTranscriptionSettingsStore(scheduledStorage) + val startupStore = StartupProcessingPolicyStore(FakeStartupStorage()) + + startupStore.save(StartupProcessingPolicy.AUTOMATIC) + + assertTrue(scheduledStore.load().enabled) + assertEquals(4, scheduledStore.load().hour) + assertEquals(30, scheduledStore.load().minute) + } + + private class FakeStartupStorage(var raw: String? = null) : StartupProcessingPolicyStorage { + override fun loadRaw(): String? = raw + + override fun saveRaw(value: String) { + raw = value + } + } + + private class FakeScheduledStorage( + private var settings: ScheduledTranscriptionSettings, + ) : ScheduledTranscriptionSettingsStorage { + override fun load(): ScheduledTranscriptionSettings = settings + + override fun save(settings: ScheduledTranscriptionSettings) { + this.settings = settings + } + } +} diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt new file mode 100644 index 0000000..1b587fc --- /dev/null +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt @@ -0,0 +1,67 @@ +package me.maxistar.voiceinbox.core + +enum class StartupProcessingPolicy { + ASK, + AUTOMATIC, + LEAVE_QUEUED, +} + +data class StartupProcessingInput( + val policy: StartupProcessingPolicy = StartupProcessingPolicy.ASK, + val evaluationComplete: Boolean = false, + val startupScanComplete: Boolean = false, + val startupScanFailed: Boolean = false, + val catalogReady: Boolean = false, + val pendingCount: Int = 0, + val failedCount: Int = 0, + val folderReady: Boolean = false, + val outputReady: Boolean = false, + val modelReady: Boolean = false, + val transcriptionStateKnown: Boolean = false, + val transcriptionActive: Boolean = false, +) + +sealed interface StartupProcessingDecision { + data object Wait : StartupProcessingDecision + + data class Prompt(val pendingCount: Int) : StartupProcessingDecision + + data class Start(val pendingCount: Int) : StartupProcessingDecision + + data class Finish(val reason: StartupProcessingFinishReason) : StartupProcessingDecision +} + +enum class StartupProcessingFinishReason { + ALREADY_EVALUATED, + SCAN_FAILED, + NO_PENDING, + LEAVE_QUEUED, +} + +object StartupProcessingRules { + fun decide(input: StartupProcessingInput): StartupProcessingDecision { + if (input.evaluationComplete) { + return StartupProcessingDecision.Finish(StartupProcessingFinishReason.ALREADY_EVALUATED) + } + if (input.startupScanFailed) { + return StartupProcessingDecision.Finish(StartupProcessingFinishReason.SCAN_FAILED) + } + if (!input.startupScanComplete || !input.catalogReady || !input.transcriptionStateKnown) { + return StartupProcessingDecision.Wait + } + if (input.pendingCount <= 0) { + return StartupProcessingDecision.Finish(StartupProcessingFinishReason.NO_PENDING) + } + if (input.policy == StartupProcessingPolicy.LEAVE_QUEUED) { + return StartupProcessingDecision.Finish(StartupProcessingFinishReason.LEAVE_QUEUED) + } + if (!input.folderReady || !input.outputReady || !input.modelReady || input.transcriptionActive) { + return StartupProcessingDecision.Wait + } + return when (input.policy) { + StartupProcessingPolicy.ASK -> StartupProcessingDecision.Prompt(input.pendingCount) + StartupProcessingPolicy.AUTOMATIC -> StartupProcessingDecision.Start(input.pendingCount) + StartupProcessingPolicy.LEAVE_QUEUED -> error("Leave-queued policy is handled above") + } + } +} diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt new file mode 100644 index 0000000..e25534e --- /dev/null +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt @@ -0,0 +1,109 @@ +package me.maxistar.voiceinbox.core + +import kotlin.test.Test +import kotlin.test.assertEquals + +class StartupProcessingRulesTest { + @Test + fun askPromptsWhenStartupStateIsReady() { + assertEquals( + StartupProcessingDecision.Prompt(3), + StartupProcessingRules.decide(readyInput(policy = StartupProcessingPolicy.ASK, pendingCount = 3)), + ) + } + + @Test + fun automaticStartsWhenStartupStateIsReady() { + assertEquals( + StartupProcessingDecision.Start(2), + StartupProcessingRules.decide(readyInput(policy = StartupProcessingPolicy.AUTOMATIC, pendingCount = 2)), + ) + } + + @Test + fun leaveQueuedFinishesWithoutProcessing() { + assertEquals( + StartupProcessingDecision.Finish(StartupProcessingFinishReason.LEAVE_QUEUED), + StartupProcessingRules.decide(readyInput(policy = StartupProcessingPolicy.LEAVE_QUEUED)), + ) + } + + @Test + fun incompleteReadinessWaits() { + val ready = readyInput() + val incomplete = listOf( + ready.copy(startupScanComplete = false), + ready.copy(catalogReady = false), + ready.copy(folderReady = false), + ready.copy(outputReady = false), + ready.copy(modelReady = false), + ready.copy(transcriptionStateKnown = false), + ) + + incomplete.forEach { input -> + assertEquals(StartupProcessingDecision.Wait, StartupProcessingRules.decide(input)) + } + } + + @Test + fun unavailablePrerequisiteCanBecomeReadyLater() { + val missingOutput = readyInput().copy(outputReady = false) + assertEquals(StartupProcessingDecision.Wait, StartupProcessingRules.decide(missingOutput)) + assertEquals( + StartupProcessingDecision.Prompt(1), + StartupProcessingRules.decide(missingOutput.copy(outputReady = true)), + ) + } + + @Test + fun emptyAndFailedOnlyCatalogsFinishAsNoPending() { + val finish = StartupProcessingDecision.Finish(StartupProcessingFinishReason.NO_PENDING) + assertEquals(finish, StartupProcessingRules.decide(readyInput(pendingCount = 0))) + assertEquals(finish, StartupProcessingRules.decide(readyInput(pendingCount = 0, failedCount = 4))) + } + + @Test + fun activeTranscriptionWaits() { + assertEquals( + StartupProcessingDecision.Wait, + StartupProcessingRules.decide(readyInput(transcriptionActive = true)), + ) + } + + @Test + fun scanFailureIsTerminal() { + assertEquals( + StartupProcessingDecision.Finish(StartupProcessingFinishReason.SCAN_FAILED), + StartupProcessingRules.decide(StartupProcessingInput(startupScanFailed = true)), + ) + } + + @Test + fun completedEvaluationIsIdempotent() { + assertEquals( + StartupProcessingDecision.Finish(StartupProcessingFinishReason.ALREADY_EVALUATED), + StartupProcessingRules.decide(readyInput(evaluationComplete = true)), + ) + } + + private fun readyInput( + policy: StartupProcessingPolicy = StartupProcessingPolicy.ASK, + pendingCount: Int = 1, + failedCount: Int = 0, + transcriptionActive: Boolean = false, + evaluationComplete: Boolean = false, + ): StartupProcessingInput = + StartupProcessingInput( + policy = policy, + evaluationComplete = evaluationComplete, + startupScanComplete = true, + catalogReady = true, + pendingCount = pendingCount, + failedCount = failedCount, + folderReady = true, + outputReady = true, + modelReady = true, + transcriptionStateKnown = true, + transcriptionActive = transcriptionActive, + ) +} diff --git a/website/src/pages/faq.astro b/website/src/pages/faq.astro index f2603de..101ce5b 100644 --- a/website/src/pages/faq.astro +++ b/website/src/pages/faq.astro @@ -55,8 +55,8 @@ import Layout from '../layouts/Layout.astro'; Is iOS supported?

iOS has an active MVP: import audio files, preview playback, speech model download, local transcription, - transcript display, and output append are available in the development app. Folder automation, scheduled background transcription, - and full Android parity are still in progress. + transcript display, selected-folder refresh, startup processing, and output append are available in the development app. + Scheduled background transcription and full Android parity are still in progress.

@@ -77,6 +77,14 @@ import Layout from '../layouts/Layout.astro';

+
+ What happens when Voice Inbox finds queued files at startup? +

+ Startup processing defaults to Ask. In Settings, you can instead make the app transcribe automatically or leave files queued. + This foreground startup behavior is separate from Android's optional nightly background transcription. +

+
+
Can scheduled transcription run exactly at my selected time?

diff --git a/website/src/pages/setup.astro b/website/src/pages/setup.astro index 0138c7f..c696d02 100644 --- a/website/src/pages/setup.astro +++ b/website/src/pages/setup.astro @@ -88,12 +88,20 @@ const releaseUrl = `${githubUrl}/releases`;

-

6. Optional: scheduled transcription

+

6. Optional: startup processing

- Open Settings to enable scheduled transcription and choose a preferred time. Android may delay background work, + In Settings, choose whether Voice Inbox should ask, transcribe automatically, or leave files queued when + the app opens and finishes scanning. Ask is the default. Startup processing is available on Android and iOS. +

+
+ +
+

7. Optional: nightly transcription

+

+ On Android, open Settings to enable nightly transcription and choose a preferred time. Android may delay background work, so scheduled runs should be treated as best-effort rather than exact alarms.

-

Scheduled background transcription is not part of the current iOS MVP.

+

Nightly background transcription is independent from startup processing and is not part of the current iOS MVP.

From b6914c1f73d83733456e2a86de8efb96472ff17c Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Thu, 16 Jul 2026 07:02:35 +0200 Subject: [PATCH 04/10] defer model loading --- .../me/maxistar/voiceinbox/MainActivity.kt | 28 ++- .../voiceinbox/NativeTranscriptionBridge.kt | 2 + .../ScheduledTranscriptionWorker.kt | 5 +- .../voiceinbox/SpeechModelDownloadWorker.kt | 1 + .../voiceinbox/SpeechModelPreparation.kt | 38 +++ .../voiceinbox/SpeechModelReadinessManager.kt | 13 +- .../voiceinbox/SpeechModelRepository.kt | 56 ++++- .../StartupProcessingCoordinator.kt | 7 +- .../voiceinbox/TranscriptionWorker.kt | 13 +- .../voiceinbox/SpeechModelPreparationTest.kt | 97 ++++++++ .../SpeechModelReadinessManagerTest.kt | 12 - .../voiceinbox/SpeechModelRepositoryTest.kt | 32 +++ iosApp/VoiceInbox.xcodeproj/project.pbxproj | 112 +++++++++ .../xcschemes/VoiceInbox.xcscheme | 27 +++ iosApp/VoiceInbox/ContentView.swift | 4 + .../VoiceInbox/IosMainScreenShellState.swift | 13 +- ...IosSingleFileTranscriptionController.swift | 84 +++++-- iosApp/VoiceInbox/IosSpeechModelStore.swift | 212 +++++++++++++++- .../DeferredSpeechModelLoadingTests.swift | 144 +++++++++++ .../core/MainScreenStateController.kt | 10 +- .../voiceinbox/core/SpeechModelLifecycle.kt | 21 ++ .../voiceinbox/core/StartupProcessingRules.kt | 4 +- .../voiceinbox/core/TranscriptionUiRules.kt | 19 +- .../core/MainScreenStateControllerTest.kt | 32 ++- .../core/StartupProcessingRulesTest.kt | 4 +- .../core/TranscriptionUiRulesTest.kt | 16 +- src/engine.rs | 226 +++++++----------- src/lib.rs | 26 +- 28 files changed, 1018 insertions(+), 240 deletions(-) create mode 100644 app/src/main/java/me/maxistar/voiceinbox/SpeechModelPreparation.kt create mode 100644 app/src/test/java/me/maxistar/voiceinbox/SpeechModelPreparationTest.kt create mode 100644 iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift create mode 100644 shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SpeechModelLifecycle.kt diff --git a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt index aa53522..9448031 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/MainActivity.kt @@ -659,8 +659,9 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen evaluateStartupProcessing() } WorkInfo.State.SUCCEEDED -> { - if (shouldHandleModelInstallSuccess(info.id.toString())) { - modelReadiness.invalidate() + if (shouldHandleModelInstallSuccess(info.id.toString())) { + modelReadiness.invalidate() + SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) } refreshModel() } @@ -736,6 +737,10 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen completedFiles = data.getInt(TranscriptionWorker.KEY_COMPLETED_FILES, 0) totalFiles = data.getInt(TranscriptionWorker.KEY_TOTAL_FILES, 0) failedFiles = data.getInt(TranscriptionWorker.KEY_FAILED_FILES, 0) + if (info.state == WorkInfo.State.FAILED && shouldRefreshModelAfterFailure(info.id.toString())) { + modelReadiness.invalidate() + refreshModel() + } if (transcriptionActive() || transcriptionFinished) { scanMessage = null statusError = null @@ -1000,10 +1005,18 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen MainScreenStateController.state( MainScreenInput( modelMessage = modelMessage, - modelLoading = modelLoading, + modelInstallationState = when { + modelLoading -> SpeechModelInstallationState.INSTALLING + modelReady -> SpeechModelInstallationState.INSTALLED + modelDownloadAvailable -> SpeechModelInstallationState.NOT_INSTALLED + else -> SpeechModelInstallationState.INVALID + }, + modelRuntimeState = if ( + transcriptionState == TranscriptionObservationState.ACTIVE && + transcriptionPhase?.contains("model", ignoreCase = true) == true + ) SpeechModelRuntimeState.LOADING else SpeechModelRuntimeState.UNLOADED, modelDownloadAvailable = modelDownloadAvailable, modelDownloadProgress = modelDownloadProgress, - modelReady = modelReady, outputSelected = outputUri != null, folderSelected = folderUri != null, pendingCount = pendingCount, @@ -1059,19 +1072,20 @@ class MainActivity : AppCompatActivity(), StartupProcessingDialogFragment.Listen @Volatile private var sharedModelReadiness: SpeechModelReadinessManager? = null private val handledModelInstallSuccessIds = mutableSetOf() + private val handledTranscriptionFailureIds = mutableSetOf() fun getSharedModelReadiness(repository: SpeechModelRepository): SpeechModelReadinessManager = sharedModelReadiness ?: synchronized(this) { sharedModelReadiness ?: SpeechModelReadinessManager( repository = repository, - initializeModel = { directory -> - NativeTranscriptionBridge.initialize(directory.absolutePath) - }, executor = Executors.newSingleThreadExecutor(), ).also { sharedModelReadiness = it } } fun shouldHandleModelInstallSuccess(id: String): Boolean = synchronized(this) { handledModelInstallSuccessIds.add(id) } + + fun shouldRefreshModelAfterFailure(id: String): Boolean = + synchronized(this) { handledTranscriptionFailureIds.add(id) } } } diff --git a/app/src/main/java/me/maxistar/voiceinbox/NativeTranscriptionBridge.kt b/app/src/main/java/me/maxistar/voiceinbox/NativeTranscriptionBridge.kt index d185283..94a920e 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/NativeTranscriptionBridge.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/NativeTranscriptionBridge.kt @@ -22,6 +22,8 @@ object NativeTranscriptionBridge { external fun initialize(modelDirectory: String): Boolean + external fun reset() + private external fun transcribeChunkJson(samples: FloatArray): String? fun transcribeChunk(samples: FloatArray): NativeChunkResult? { diff --git a/app/src/main/java/me/maxistar/voiceinbox/ScheduledTranscriptionWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/ScheduledTranscriptionWorker.kt index a8ce7f4..155cd7a 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/ScheduledTranscriptionWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/ScheduledTranscriptionWorker.kt @@ -51,10 +51,9 @@ class ScheduledTranscriptionWorker( documentAccess.requireAppendable(output) folderScanner.requireReadable(folder) - val model = SpeechModelRepository( + if (SpeechModelRepository( applicationContext.noBackupFilesDir.resolve("models"), - ).inspect() as? InstalledSpeechModelState.Ready ?: return - if (!NativeTranscriptionBridge.initialize(model.directory.absolutePath)) return + ).inspectLightweight() !is InstalledSpeechModelState.Ready) return val catalog = AndroidSqlDelightAudioCatalogFactory(applicationContext).create() val files = folderScanner.scan(folder) diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt index 0183b63..ed5a918 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelDownloadWorker.kt @@ -85,6 +85,7 @@ class SpeechModelDownloadWorker( val installedDirectory = repository.activate().getOrElse { return failure(it.message ?: "Failed to activate speech model") } + SpeechModelPreparation.invalidate(NativeTranscriptionBridge::reset) return Result.success(workDataOf(KEY_MODEL_PATH to installedDirectory.absolutePath)) } diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelPreparation.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelPreparation.kt new file mode 100644 index 0000000..87e7a71 --- /dev/null +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelPreparation.kt @@ -0,0 +1,38 @@ +package me.maxistar.voiceinbox + +import java.io.File + +object SpeechModelPreparation { + private val lock = Any() + private var preparedDirectory: String? = null + + fun prepare( + repository: SpeechModelRepository, + initializeModel: (File) -> Boolean, + ): Result = synchronized(lock) { + val expected = repository.installedDirectory.canonicalPath + if (preparedDirectory == expected) { + return@synchronized Result.success(repository.installedDirectory) + } + runCatching { + val installed = repository.inspect() + check(installed is InstalledSpeechModelState.Ready) { + when (installed) { + InstalledSpeechModelState.Missing -> "Speech model is not installed" + is InstalledSpeechModelState.Invalid -> installed.reason + is InstalledSpeechModelState.Ready -> error("unreachable") + } + } + check(initializeModel(installed.directory)) { "Speech model failed to load" } + preparedDirectory = installed.directory.canonicalPath + installed.directory + } + } + + fun invalidate(resetNative: () -> Unit = {}) { + synchronized(lock) { + preparedDirectory = null + resetNative() + } + } +} diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelReadinessManager.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelReadinessManager.kt index 4764992..a4a886d 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelReadinessManager.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelReadinessManager.kt @@ -13,7 +13,6 @@ sealed interface SpeechModelReadinessState { class SpeechModelReadinessManager( private val repository: SpeechModelRepository, - private val initializeModel: (File) -> Boolean, private val executor: Executor, ) { private val lock = Any() @@ -53,19 +52,13 @@ class SpeechModelReadinessManager( private fun checkModel() { val state = runCatching { repository.cleanupStaleState() - when (val installed = repository.inspect()) { - is InstalledSpeechModelState.Ready -> { - if (initializeModel(installed.directory)) { - SpeechModelReadinessState.Ready(installed.directory) - } else { - SpeechModelReadinessState.Failed("Speech model failed to load") - } - } + when (val installed = repository.inspectLightweight()) { + is InstalledSpeechModelState.Ready -> SpeechModelReadinessState.Ready(installed.directory) InstalledSpeechModelState.Missing -> SpeechModelReadinessState.Missing is InstalledSpeechModelState.Invalid -> SpeechModelReadinessState.Invalid(installed.reason) } }.getOrElse { error -> - SpeechModelReadinessState.Failed(error.message ?: "Speech model failed to load") + SpeechModelReadinessState.Failed(error.message ?: "Speech model inspection failed") } val pending = synchronized(lock) { diff --git a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelRepository.kt b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelRepository.kt index 562a616..a25bba7 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/SpeechModelRepository.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/SpeechModelRepository.kt @@ -10,7 +10,15 @@ import java.nio.file.StandardCopyOption import java.util.UUID sealed interface InstalledSpeechModelState { - data class Ready(val directory: File) : InstalledSpeechModelState + data class Ready( + val directory: File, + val verification: Verification = Verification.VERIFIED, + ) : InstalledSpeechModelState { + enum class Verification { + VERIFIED, + LEGACY_UNVERIFIED, + } + } data object Missing : InstalledSpeechModelState data class Invalid(val reason: String) : InstalledSpeechModelState } @@ -23,6 +31,7 @@ class SpeechModelRepository( private val stagingRoot = File(root, "staging") private val installedRoot = File(root, "installed") private val activeVersionFile = File(root, "active-model") + private val invalidModelFile = File(root, "invalid-model") val stagingDirectory: File get() = File(stagingRoot, manifest.version) @@ -30,18 +39,39 @@ class SpeechModelRepository( val installedDirectory: File get() = File(installedRoot, manifest.version) + fun inspectLightweight(): InstalledSpeechModelState { + invalidModelFile.takeIf(File::isFile)?.readText()?.trim()?.takeIf(String::isNotEmpty)?.let { + return InstalledSpeechModelState.Invalid(it) + } + if (!installedDirectory.isDirectory) return InstalledSpeechModelState.Missing + val missing = manifest.files.firstOrNull { !File(installedDirectory, it.name).isFile } + return if (missing == null) { + InstalledSpeechModelState.Ready( + directory = installedDirectory, + verification = if (activeVersionFile.takeIf(File::isFile)?.readText()?.trim() == manifest.version) { + InstalledSpeechModelState.Ready.Verification.VERIFIED + } else { + InstalledSpeechModelState.Ready.Verification.LEGACY_UNVERIFIED + }, + ) + } else { + InstalledSpeechModelState.Invalid("${missing.name} is missing") + } + } + fun inspect(): InstalledSpeechModelState { val activeVersion = activeVersionFile.takeIf(File::isFile)?.readText()?.trim() if (activeVersion == manifest.version) { - return validateDirectory(installedDirectory) + return recordValidation(validateDirectory(installedDirectory)) } return when (val installed = validateDirectory(installedDirectory)) { is InstalledSpeechModelState.Ready -> { writeActiveVersion(manifest.version) + invalidModelFile.delete() installed } - is InstalledSpeechModelState.Invalid -> installed + is InstalledSpeechModelState.Invalid -> installed.also { writeInvalidReason(it.reason) } InstalledSpeechModelState.Missing -> InstalledSpeechModelState.Missing } } @@ -99,6 +129,7 @@ class SpeechModelRepository( "Failed to activate downloaded model" } writeActiveVersion(manifest.version) + invalidModelFile.delete() installedRoot.listFiles() ?.filter { it.name != manifest.version } @@ -134,7 +165,10 @@ class SpeechModelRepository( ) } } - return InstalledSpeechModelState.Ready(directory) + return InstalledSpeechModelState.Ready( + directory = directory, + verification = InstalledSpeechModelState.Ready.Verification.VERIFIED, + ) } private fun isValidFile(file: File, entry: SpeechModelFile): Boolean = @@ -157,6 +191,20 @@ class SpeechModelRepository( } } + private fun recordValidation(state: InstalledSpeechModelState): InstalledSpeechModelState { + when (state) { + is InstalledSpeechModelState.Ready -> invalidModelFile.delete() + is InstalledSpeechModelState.Invalid -> writeInvalidReason(state.reason) + InstalledSpeechModelState.Missing -> Unit + } + return state + } + + private fun writeInvalidReason(reason: String) { + root.mkdirs() + invalidModelFile.writeText(reason) + } + private fun cleanupTemporaryFiles(directory: File) { directory.listFiles()?.forEach { file -> if (file.isDirectory) { diff --git a/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt index a2adfa6..596710a 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/StartupProcessingCoordinator.kt @@ -4,6 +4,7 @@ import me.maxistar.voiceinbox.core.StartupProcessingDecision import me.maxistar.voiceinbox.core.StartupProcessingInput import me.maxistar.voiceinbox.core.StartupProcessingPolicy import me.maxistar.voiceinbox.core.StartupProcessingRules +import me.maxistar.voiceinbox.core.SpeechModelInstallationState class StartupProcessingCoordinator private constructor( private var stage: Stage, @@ -94,7 +95,11 @@ class StartupProcessingCoordinator private constructor( failedCount = failedCount, folderReady = folderReady, outputReady = outputReady, - modelReady = modelReady, + modelInstallationState = if (modelReady) { + SpeechModelInstallationState.INSTALLED + } else { + SpeechModelInstallationState.NOT_INSTALLED + }, transcriptionStateKnown = transcriptionStateKnown, transcriptionActive = transcriptionActive, ), diff --git a/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt b/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt index a140343..65fdbcb 100644 --- a/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt +++ b/app/src/main/java/me/maxistar/voiceinbox/TranscriptionWorker.kt @@ -34,14 +34,13 @@ class TranscriptionWorker( try { setForeground(foreground("Preparing transcription", 0, true)) - val model = SpeechModelRepository( + val modelRepository = SpeechModelRepository( applicationContext.noBackupFilesDir.resolve("models"), - ).inspect() as? InstalledSpeechModelState.Ready - ?: return@withContext failure("Speech model is not installed") - publish("Loading model", null, 0, 0, null, null) - if (!NativeTranscriptionBridge.initialize(model.directory.absolutePath)) { - return@withContext failure("Speech model failed to load") - } + ) + publish("Preparing speech model", null, 0, 0, null, null) + SpeechModelPreparation.prepare(modelRepository) { directory -> + NativeTranscriptionBridge.initialize(directory.absolutePath) + }.getOrElse { return@withContext failure(it.message ?: "Speech model preparation failed") } val batch = BatchTranscriptionUseCase( catalog = catalog, diff --git a/app/src/test/java/me/maxistar/voiceinbox/SpeechModelPreparationTest.kt b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelPreparationTest.kt new file mode 100644 index 0000000..1b38d88 --- /dev/null +++ b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelPreparationTest.kt @@ -0,0 +1,97 @@ +package me.maxistar.voiceinbox + +import me.maxistar.voiceinbox.core.SpeechModelFile +import me.maxistar.voiceinbox.core.SpeechModelManifest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.security.MessageDigest + +class SpeechModelPreparationTest { + @get:Rule + val temporaryFolder = TemporaryFolder() + + @Test + fun preparationVerifiesAndInitializesOnceUntilInvalidated() { + val repository = readyRepository() + var initializations = 0 + SpeechModelPreparation.invalidate() + + repeat(2) { + assertTrue( + SpeechModelPreparation.prepare(repository) { + initializations += 1 + true + }.isSuccess, + ) + } + assertEquals(1, initializations) + + SpeechModelPreparation.invalidate() + assertTrue( + SpeechModelPreparation.prepare(repository) { + initializations += 1 + true + }.isSuccess, + ) + assertEquals(2, initializations) + } + + @Test + fun invalidModelFailsBeforeNativeInitialization() { + val repository = readyRepository() + repository.installedDirectory.resolve("model.bin").writeText("other") + var initialized = false + SpeechModelPreparation.invalidate() + + val result = SpeechModelPreparation.prepare(repository) { + initialized = true + true + } + + assertTrue(result.isFailure) + assertTrue(!initialized) + } + + private fun readyRepository(): SpeechModelRepository { + val repository = SpeechModelRepository( + File(temporaryFolder.root, "models"), + testManifest, + usableSpace = { Long.MAX_VALUE }, + ) + repository.prepareForInstall().getOrThrow() + testFiles.forEach { (name, contents) -> + repository.stagingDirectory.resolve(name).apply { + parentFile?.mkdirs() + writeBytes(contents) + } + } + repository.activate().getOrThrow() + return repository + } + + companion object { + private val testFiles = linkedMapOf( + "model.bin" to "model".toByteArray(), + "config.json" to "{}".toByteArray(), + ) + private val testManifest = SpeechModelManifest( + modelId = "example/model", + version = "test-version", + repositoryRevision = "0123456789abcdef", + files = testFiles.map { (name, contents) -> + SpeechModelFile( + name, + contents.size.toLong(), + MessageDigest.getInstance("SHA-256") + .digest(contents) + .joinToString("") { "%02x".format(it) }, + ) + }, + safetyMarginBytes = 8, + ) + } +} diff --git a/app/src/test/java/me/maxistar/voiceinbox/SpeechModelReadinessManagerTest.kt b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelReadinessManagerTest.kt index ec95103..8669454 100644 --- a/app/src/test/java/me/maxistar/voiceinbox/SpeechModelReadinessManagerTest.kt +++ b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelReadinessManagerTest.kt @@ -19,13 +19,8 @@ class SpeechModelReadinessManagerTest { fun duplicateRefreshesAreCoalescedWhileCheckIsRunning() { val executor = QueueingExecutor() val repository = readyRepository() - var initializations = 0 val manager = SpeechModelReadinessManager( repository = repository, - initializeModel = { - initializations += 1 - true - }, executor = executor, ) val firstStates = mutableListOf() @@ -40,7 +35,6 @@ class SpeechModelReadinessManagerTest { executor.runNext() - assertEquals(1, initializations) assertTrue(firstStates.last() is SpeechModelReadinessState.Ready) assertTrue(secondStates.last() is SpeechModelReadinessState.Ready) } @@ -49,13 +43,8 @@ class SpeechModelReadinessManagerTest { fun readyStateIsCachedForLaterRefreshes() { val executor = QueueingExecutor() val repository = readyRepository() - var initializations = 0 val manager = SpeechModelReadinessManager( repository = repository, - initializeModel = { - initializations += 1 - true - }, executor = executor, ) val firstStates = mutableListOf() @@ -65,7 +54,6 @@ class SpeechModelReadinessManagerTest { executor.runNext() manager.refresh { cachedStates += it } - assertEquals(1, initializations) assertEquals(0, executor.pendingCount) assertTrue(firstStates.last() is SpeechModelReadinessState.Ready) assertEquals(1, cachedStates.size) diff --git a/app/src/test/java/me/maxistar/voiceinbox/SpeechModelRepositoryTest.kt b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelRepositoryTest.kt index 1e668b1..51889d5 100644 --- a/app/src/test/java/me/maxistar/voiceinbox/SpeechModelRepositoryTest.kt +++ b/app/src/test/java/me/maxistar/voiceinbox/SpeechModelRepositoryTest.kt @@ -37,6 +37,38 @@ class SpeechModelRepositoryTest { ) } + @Test + fun lightweightInspectionDoesNotHashInstalledPayloads() { + val repository = repository() + repository.prepareForInstall().getOrThrow() + writeValidStaging(repository) + val installed = repository.activate().getOrThrow() + installed.resolve("model.bin").writeText("other") + + val lightweight = repository.inspectLightweight() as InstalledSpeechModelState.Ready + assertEquals(InstalledSpeechModelState.Ready.Verification.VERIFIED, lightweight.verification) + assertTrue(repository.inspect() is InstalledSpeechModelState.Invalid) + assertTrue(repository.inspectLightweight() is InstalledSpeechModelState.Invalid) + } + + @Test + fun legacyInstallationWithoutReceiptIsAvailableLightweight() { + val repository = repository() + repository.prepareForInstall().getOrThrow() + writeValidStaging(repository) + repository.activate().getOrThrow() + File(temporaryFolder.root, "models/active-model").delete() + + val legacy = repository.inspectLightweight() as InstalledSpeechModelState.Ready + assertEquals(InstalledSpeechModelState.Ready.Verification.LEGACY_UNVERIFIED, legacy.verification) + assertTrue(repository.inspect() is InstalledSpeechModelState.Ready) + assertEquals( + InstalledSpeechModelState.Ready.Verification.VERIFIED, + (repository.inspectLightweight() as InstalledSpeechModelState.Ready).verification, + ) + assertTrue(File(temporaryFolder.root, "models/active-model").isFile) + } + @Test fun corruptAndIncompleteModelsAreRejected() { val repository = repository() diff --git a/iosApp/VoiceInbox.xcodeproj/project.pbxproj b/iosApp/VoiceInbox.xcodeproj/project.pbxproj index 1a3cb12..559a8da 100644 --- a/iosApp/VoiceInbox.xcodeproj/project.pbxproj +++ b/iosApp/VoiceInbox.xcodeproj/project.pbxproj @@ -23,6 +23,7 @@ 361EE03677F4524C001986C0 /* ShareViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = A2A88E9CFF2023E878C6EC22 /* ShareViewController.swift */; }; 5E50C86D75D247316B5953AB /* VoiceInboxShareExtension.appex in Embed App Extensions */ = {isa = PBXBuildFile; fileRef = 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 9FAC76B26E6BD4116D968475 /* IosSharedImportStaging.swift in Sources */ = {isa = PBXBuildFile; fileRef = CD030301C0E14C9932DCBFE6 /* IosSharedImportStaging.swift */; }; + A10000000000000000000001 /* DeferredSpeechModelLoadingTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = A10000000000000000000002 /* DeferredSpeechModelLoadingTests.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -33,6 +34,13 @@ remoteGlobalIDString = D93524F1DBD24C87F8DF0EB9; remoteInfo = VoiceInboxShareExtension; }; + A1000000000000000000000A /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 101010101010101010101010 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 101010101010101010101007; + remoteInfo = VoiceInbox; + }; /* End PBXContainerItemProxy section */ /* Begin PBXCopyFilesBuildPhase section */ @@ -70,6 +78,8 @@ CD030301C0E14C9932DCBFE6 /* IosSharedImportStaging.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; name = IosSharedImportStaging.swift; path = ../VoiceInbox/IosSharedImportStaging.swift; sourceTree = ""; }; EF9FEAB9A5BF00B4E66B996C /* VoiceInbox.entitlements */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.entitlements; path = VoiceInbox.entitlements; sourceTree = ""; }; F63411837D138D0E90F5BA1D /* Info.plist */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + A10000000000000000000002 /* DeferredSpeechModelLoadingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DeferredSpeechModelLoadingTests.swift; sourceTree = ""; }; + A10000000000000000000003 /* VoiceInboxTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = VoiceInboxTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -87,6 +97,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000004 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -96,6 +113,7 @@ 101010101010101010101013 /* VoiceInbox */, 101010101010101010101006 /* Products */, F139417D5AA7BD6EFD2ECADD /* VoiceInboxShareExtension */, + A10000000000000000000005 /* VoiceInboxTests */, ); sourceTree = ""; }; @@ -104,6 +122,7 @@ children = ( 101010101010101010101003 /* VoiceInbox.app */, 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */, + A10000000000000000000003 /* VoiceInboxTests.xctest */, ); name = Products; sourceTree = ""; @@ -140,6 +159,14 @@ path = VoiceInboxShareExtension; sourceTree = ""; }; + A10000000000000000000005 /* VoiceInboxTests */ = { + isa = PBXGroup; + children = ( + A10000000000000000000002 /* DeferredSpeechModelLoadingTests.swift */, + ); + path = VoiceInboxTests; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -181,6 +208,24 @@ productReference = 242890FE4BBC56636FBA9F10 /* VoiceInboxShareExtension.appex */; productType = "com.apple.product-type.app-extension"; }; + A10000000000000000000006 /* VoiceInboxTests */ = { + isa = PBXNativeTarget; + buildConfigurationList = A1000000000000000000000D /* Build configuration list for PBXNativeTarget "VoiceInboxTests" */; + buildPhases = ( + A10000000000000000000008 /* Sources */, + A10000000000000000000004 /* Frameworks */, + A10000000000000000000007 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + A10000000000000000000009 /* PBXTargetDependency */, + ); + name = VoiceInboxTests; + productName = VoiceInboxTests; + productReference = A10000000000000000000003 /* VoiceInboxTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -194,6 +239,9 @@ 101010101010101010101007 = { CreatedOnToolsVersion = 16.0; }; + A10000000000000000000006 = { + CreatedOnToolsVersion = 16.0; + }; }; }; buildConfigurationList = 101010101010101010101016 /* Build configuration list for PBXProject "VoiceInbox" */; @@ -211,6 +259,7 @@ targets = ( 101010101010101010101007 /* VoiceInbox */, D93524F1DBD24C87F8DF0EB9 /* VoiceInboxShareExtension */, + A10000000000000000000006 /* VoiceInboxTests */, ); }; /* End PBXProject section */ @@ -231,6 +280,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000007 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ @@ -307,6 +363,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + A10000000000000000000008 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + A10000000000000000000001 /* DeferredSpeechModelLoadingTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -316,6 +380,11 @@ target = D93524F1DBD24C87F8DF0EB9 /* VoiceInboxShareExtension */; targetProxy = B02BB42352D5BAE85F16E7A7 /* PBXContainerItemProxy */; }; + A10000000000000000000009 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 101010101010101010101007 /* VoiceInbox */; + targetProxy = A1000000000000000000000A /* PBXContainerItemProxy */; + }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ @@ -581,6 +650,40 @@ }; name = Release; }; + A1000000000000000000000B /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = 2835P3MJR8; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + PRODUCT_BUNDLE_IDENTIFIER = me.maxistar.voiceinbox.ios.tests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/VoiceInbox.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/VoiceInbox"; + }; + name = Debug; + }; + A1000000000000000000000C /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = 2835P3MJR8; + GENERATE_INFOPLIST_FILE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.0; + PRODUCT_BUNDLE_IDENTIFIER = me.maxistar.voiceinbox.ios.tests; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/VoiceInbox.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/VoiceInbox"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -611,6 +714,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + A1000000000000000000000D /* Build configuration list for PBXNativeTarget "VoiceInboxTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + A1000000000000000000000B /* Debug */, + A1000000000000000000000C /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; /* End XCConfigurationList section */ }; rootObject = 101010101010101010101010 /* Project object */; diff --git a/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme index a3ade6b..ea1695e 100644 --- a/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme +++ b/iosApp/VoiceInbox.xcodeproj/xcshareddata/xcschemes/VoiceInbox.xcscheme @@ -21,6 +21,20 @@ ReferencedContainer = "container:VoiceInbox.xcodeproj"> + + + + + + + + + + Bool, + claim: () -> Void + ) async -> Bool { + guard await prepare() else { return false } + claim() + return true + } +} + @_silgen_name("voiceinbox_transcription_initialize") private func voiceinbox_transcription_initialize(_ modelDirectory: UnsafePointer) -> Bool @@ -20,6 +32,9 @@ private func voiceinbox_transcription_string_free(_ value: UnsafeMutablePointer< @_silgen_name("voiceinbox_transcription_backend_configured") private func voiceinbox_transcription_backend_configured() -> Bool +@_silgen_name("voiceinbox_transcription_reset") +private func voiceinbox_transcription_reset() + struct IosSingleFileTranscriptionState { let active: Bool let fileId: Int64? @@ -80,17 +95,17 @@ final class IosSingleFileTranscriptionController: ObservableObject { file: IosImportedAudioFile, localURL: URL, modelDirectory: URL, + modelStore: IosSpeechModelStore, outputDocument: IosSelectedOutputDocument, store: IosAudioImportStore, onSuccess: ((String) -> Void)? = nil ) { task?.cancel() - store.markProcessing(fileId: file.id) state = IosSingleFileTranscriptionState( active: true, fileId: file.id, fileName: file.displayName, - phase: SingleFileTranscriptionUseCase.companion.PHASE_DECODING_AUDIO, + phase: "Preparing speech model", processedUs: 0, durationUs: 0, completedFiles: 0, @@ -102,12 +117,26 @@ final class IosSingleFileTranscriptionController: ObservableObject { message = nil task = Task { + let prepared = await IosTranscriptionPreparationGate.prepareAndClaim( + prepare: { await modelStore.prepareForTranscription() != nil }, + claim: { store.markProcessing(fileId: file.id) } + ) + guard prepared else { + message = modelStore.message ?? "Speech model preparation failed." + state = .idle + return + } + guard !Task.isCancelled else { + state = .idle + return + } let outcome = await Task.detached(priority: .userInitiated) { Self.runSharedTranscription( file: file, localURL: localURL, modelDirectory: modelDirectory.path, - outputDocument: outputDocument + outputDocument: outputDocument, + modelPrepared: true ) { progress in Task { @MainActor in self.state = IosSingleFileTranscriptionState( @@ -152,6 +181,7 @@ final class IosSingleFileTranscriptionController: ObservableObject { func transcribeAll( modelDirectory: URL, + modelStore: IosSpeechModelStore, outputDocument: IosSelectedOutputDocument, store: IosAudioImportStore, onFinished: (() -> Void)? = nil @@ -162,7 +192,7 @@ final class IosSingleFileTranscriptionController: ObservableObject { active: true, fileId: nil, fileName: nil, - phase: "Preparing transcription", + phase: "Preparing speech model", processedUs: 0, durationUs: 0, completedFiles: 0, @@ -174,6 +204,15 @@ final class IosSingleFileTranscriptionController: ObservableObject { message = nil task = Task { + guard await modelStore.prepareForTranscription() != nil else { + message = modelStore.message ?? "Speech model preparation failed." + state = .idle + return + } + guard !Task.isCancelled else { + state = .idle + return + } let outcome = await Task.detached(priority: .userInitiated) { Self.runSharedBatchTranscription( modelDirectory: modelDirectory.path, @@ -225,6 +264,7 @@ final class IosSingleFileTranscriptionController: ObservableObject { file: IosImportedAudioFile, localURL: URL, modelDirectory: URL, + modelStore: IosSpeechModelStore, outputDocument: IosSelectedOutputDocument, store: IosAudioImportStore, onSuccess: ((String) -> Void)? = nil @@ -234,6 +274,7 @@ final class IosSingleFileTranscriptionController: ObservableObject { file: file, localURL: localURL, modelDirectory: modelDirectory, + modelStore: modelStore, outputDocument: outputDocument, store: store, onSuccess: onSuccess @@ -245,11 +286,12 @@ final class IosSingleFileTranscriptionController: ObservableObject { localURL: URL, modelDirectory: String, outputDocument: IosSelectedOutputDocument, + modelPrepared: Bool = false, onProgress: @escaping (SingleFileTranscriptionProgress) -> Void ) -> SingleFileTranscriptionOutcome { let decoder = IosPlatformAudioDecoder(localURL: localURL) let transcriber = IosNativeTranscriber(modelDirectory: modelDirectory) - guard transcriber.initialize(modelDirectory: transcriber.modelDirectory) else { + guard modelPrepared || transcriber.initialize(modelDirectory: transcriber.modelDirectory) else { return SingleFileTranscriptionOutcome( success: false, result: nil, @@ -345,6 +387,7 @@ private final class IosBatchEntryOutcomeTranscriber: OutcomeBatchEntryTranscribe localURL: Self.localURL(documentUri: entry.documentUri), modelDirectory: modelDirectory, outputDocument: outputDocument, + modelPrepared: true, onProgress: onProgress ) } @@ -477,7 +520,7 @@ private final class IosPlatformAudioDecoder: PlatformAudioDecoder { } } -private final class IosNativeTranscriber: PlatformNativeTranscriber { +final class IosNativeTranscriber: PlatformNativeTranscriber { let modelDirectory: String private(set) var lastError: String? @@ -489,12 +532,24 @@ private final class IosNativeTranscriber: PlatformNativeTranscriber { voiceinbox_transcription_backend_configured() } + static func prepare(modelDirectory: String) -> Bool { + modelDirectory.withCString { voiceinbox_transcription_initialize($0) } + } + + static func resetModel() { + voiceinbox_transcription_reset() + } + + static func consumeLastError() -> String? { + guard let pointer = voiceinbox_transcription_last_error() else { return nil } + defer { voiceinbox_transcription_string_free(pointer) } + return String(cString: pointer) + } + func initialize(modelDirectory: String) -> Bool { - let ok = modelDirectory.withCString { pointer in - voiceinbox_transcription_initialize(pointer) - } + let ok = Self.prepare(modelDirectory: modelDirectory) if !ok { - lastError = Self.consumeNativeError() + lastError = Self.consumeLastError() ?? "iOS native transcription failed." } return ok } @@ -510,7 +565,7 @@ private final class IosNativeTranscriber: PlatformNativeTranscriber { voiceinbox_transcription_transcribe_chunk_json(pointer.baseAddress, pointer.count) } guard let result else { - lastError = Self.consumeNativeError() + lastError = Self.consumeLastError() ?? "iOS native transcription failed." return nil } defer { voiceinbox_transcription_string_free(result) } @@ -526,13 +581,6 @@ private final class IosNativeTranscriber: PlatformNativeTranscriber { return text } - private static func consumeNativeError() -> String { - guard let pointer = voiceinbox_transcription_last_error() else { - return "iOS native transcription failed." - } - defer { voiceinbox_transcription_string_free(pointer) } - return String(cString: pointer) - } } private final class IosTranscriptStaging: PlatformTranscriptStaging { diff --git a/iosApp/VoiceInbox/IosSpeechModelStore.swift b/iosApp/VoiceInbox/IosSpeechModelStore.swift index 3a404b0..e70e70d 100644 --- a/iosApp/VoiceInbox/IosSpeechModelStore.swift +++ b/iosApp/VoiceInbox/IosSpeechModelStore.swift @@ -3,11 +3,22 @@ import CryptoKit import Foundation import Shared +enum IosSpeechModelInstallationState: Equatable { + case missing + case installedVerified + case installedLegacy + case invalid +} + struct IosSpeechModelStatus { let directory: URL - let isReady: Bool + let installationState: IosSpeechModelInstallationState let missingFiles: [String] + var isReady: Bool { + installationState == .installedVerified || installationState == .installedLegacy + } + var summary: String { if isReady { return "Speech model installed" @@ -48,6 +59,14 @@ enum IosSpeechModelPaths { static var backupDirectory: URL { applicationSupportDirectory.appendingPathComponent("SpeechModel.previous", isDirectory: true) } + + static var receiptFile: URL { + applicationSupportDirectory.appendingPathComponent("SpeechModel.receipt") + } + + static var invalidFile: URL { + applicationSupportDirectory.appendingPathComponent("SpeechModel.invalid") + } } struct IosSpeechModelDownloadProgress { @@ -66,13 +85,52 @@ final class IosSpeechModelStore: ObservableObject { @Published private(set) var status: IosSpeechModelStatus @Published private(set) var isInstalling = false @Published private(set) var downloadProgress: IosSpeechModelDownloadProgress? + @Published private(set) var runtimeState = SpeechModelRuntimeState.unloaded @Published var message: String? private var downloadTask: Task? + private var preparationTask: Task? + private let installationDirectory: URL + private let inspectInstallation: @Sendable (URL) -> IosSpeechModelStatus + private let validateInstallation: @Sendable (URL) -> [String] + private let prepareNative: @Sendable (String) -> Bool + private let nativeError: @Sendable () -> String? + private let recordVerified: @Sendable () -> Void + private let recordInvalid: @Sendable (String) -> Void + private let resetNative: @Sendable () -> Void + + convenience init() { + self.init( + directory: IosSpeechModelPaths.modelDirectory, + inspectInstallation: { Self.inspectLightweight(directory: $0) }, + validateInstallation: { Self.validateModelFiles(in: $0).missingFiles }, + prepareNative: { IosNativeTranscriber.prepare(modelDirectory: $0) }, + nativeError: { IosNativeTranscriber.consumeLastError() }, + recordVerified: { Self.recordVerifiedInstallation() }, + recordInvalid: { Self.recordInvalidInstallation($0) }, + resetNative: { IosNativeTranscriber.resetModel() } + ) + } - init() { - let directory = IosSpeechModelPaths.modelDirectory - status = Self.validate(directory: directory) + init( + directory: URL, + inspectInstallation: @escaping @Sendable (URL) -> IosSpeechModelStatus, + validateInstallation: @escaping @Sendable (URL) -> [String], + prepareNative: @escaping @Sendable (String) -> Bool, + nativeError: @escaping @Sendable () -> String?, + recordVerified: @escaping @Sendable () -> Void, + recordInvalid: @escaping @Sendable (String) -> Void, + resetNative: @escaping @Sendable () -> Void + ) { + installationDirectory = directory + self.inspectInstallation = inspectInstallation + self.validateInstallation = validateInstallation + self.prepareNative = prepareNative + self.nativeError = nativeError + self.recordVerified = recordVerified + self.recordInvalid = recordInvalid + self.resetNative = resetNative + status = inspectInstallation(directory) } var isReady: Bool { @@ -88,7 +146,7 @@ final class IosSpeechModelStore: ObservableObject { } func reload() { - status = Self.validate(directory: IosSpeechModelPaths.modelDirectory) + status = inspectInstallation(installationDirectory) } func installModel(from sourceURL: URL) { @@ -104,7 +162,8 @@ final class IosSpeechModelStore: ObservableObject { }.value isInstalling = false - status = Self.validate(directory: IosSpeechModelPaths.modelDirectory) + status = Self.inspectLightweight(directory: IosSpeechModelPaths.modelDirectory) + runtimeState = .unloaded message = result.message } } @@ -134,7 +193,8 @@ final class IosSpeechModelStore: ObservableObject { isInstalling = false downloadTask = nil - status = Self.validate(directory: IosSpeechModelPaths.modelDirectory) + status = Self.inspectLightweight(directory: IosSpeechModelPaths.modelDirectory) + runtimeState = .unloaded downloadProgress = nil message = result.message } @@ -148,13 +208,130 @@ final class IosSpeechModelStore: ObservableObject { message = "Speech model download cancelled." } - nonisolated private static func validate(directory: URL) -> IosSpeechModelStatus { - let result = validateModelFiles(in: directory) + func prepareForTranscription() async -> URL? { + if runtimeState == .loaded { return modelDirectory } + let task: Task + if let existing = preparationTask { + task = existing + } else { + runtimeState = .loading + message = "Preparing speech model..." + let directory = modelDirectory + let validateInstallation = validateInstallation + let prepareNative = prepareNative + let nativeError = nativeError + let recordVerified = recordVerified + let recordInvalid = recordInvalid + task = Task.detached(priority: .userInitiated) { + let validationIssues = validateInstallation(directory) + guard validationIssues.isEmpty else { + let reason = validationIssues.joined(separator: ", ") + recordInvalid(reason) + return PreparationResult( + directory: nil, + invalidInstallation: true, + message: "Speech model is invalid: \(reason)" + ) + } + guard prepareNative(directory.path) else { + return PreparationResult( + directory: nil, + invalidInstallation: false, + message: nativeError() + ?? "Speech model failed to load." + ) + } + recordVerified() + return PreparationResult(directory: directory, invalidInstallation: false, message: nil) + } + preparationTask = task + } + + let result = await task.value + preparationTask = nil + if let directory = result.directory { + status = inspectInstallation(directory) + runtimeState = .loaded + message = nil + return directory + } + runtimeState = .failed + if result.invalidInstallation { + status = inspectInstallation(modelDirectory) + } + message = result.message + return nil + } + + func invalidateRuntimeAfterReplacement() { + preparationTask?.cancel() + preparationTask = nil + runtimeState = .unloaded + resetNative() + } + + nonisolated static func inspectLightweight( + directory: URL, + receiptFile: URL = IosSpeechModelPaths.receiptFile, + invalidFile: URL = IosSpeechModelPaths.invalidFile, + requiredFileNames: [String]? = nil + ) -> IosSpeechModelStatus { + if let reason = try? String(contentsOf: invalidFile, encoding: .utf8), + !reason.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return IosSpeechModelStatus( + directory: directory, + installationState: .invalid, + missingFiles: [reason] + ) + } + let fileManager = FileManager.default + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: directory.path, isDirectory: &isDirectory), isDirectory.boolValue else { + return IosSpeechModelStatus( + directory: directory, + installationState: .missing, + missingFiles: ["model directory"] + ) + } + let missing = (requiredFileNames ?? manifestFiles().map(\.name)).compactMap { fileName in + fileManager.isReadableFile(atPath: directory.appendingPathComponent(fileName).path) + ? nil + : fileName + } + guard missing.isEmpty else { + return IosSpeechModelStatus( + directory: directory, + installationState: .invalid, + missingFiles: missing + ) + } + let receipt = try? String(contentsOf: receiptFile, encoding: .utf8) + let state: IosSpeechModelInstallationState = receipt?.trimmingCharacters(in: .whitespacesAndNewlines) + == EmbeddedSpeechModel.shared.manifest.version + ? .installedVerified + : .installedLegacy return IosSpeechModelStatus( directory: directory, - isReady: result.missingFiles.isEmpty, - missingFiles: result.missingFiles + installationState: state, + missingFiles: [] + ) + } + + nonisolated private static func recordVerifiedInstallation() { + try? FileManager.default.createDirectory( + at: IosSpeechModelPaths.applicationSupportDirectory, + withIntermediateDirectories: true + ) + try? EmbeddedSpeechModel.shared.manifest.version.write( + to: IosSpeechModelPaths.receiptFile, + atomically: true, + encoding: .utf8 ) + try? removeItemIfExists(IosSpeechModelPaths.invalidFile) + } + + nonisolated private static func recordInvalidInstallation(_ reason: String) { + try? reason.write(to: IosSpeechModelPaths.invalidFile, atomically: true, encoding: .utf8) } nonisolated private static func validateModelFiles(in directory: URL) -> ModelValidationResult { @@ -221,6 +398,8 @@ final class IosSpeechModelStore: ObservableObject { do { try fileManager.moveItem(at: IosSpeechModelPaths.installDirectory, to: IosSpeechModelPaths.modelDirectory) try removeItemIfExists(IosSpeechModelPaths.backupDirectory) + recordVerifiedInstallation() + IosNativeTranscriber.resetModel() return InstallResult(message: "Speech model installed.") } catch { if fileManager.fileExists(atPath: IosSpeechModelPaths.backupDirectory.path) { @@ -311,6 +490,8 @@ final class IosSpeechModelStore: ObservableObject { } try activateStagedModel() + recordVerifiedInstallation() + IosNativeTranscriber.resetModel() return InstallResult(message: "Speech model downloaded and installed.") } catch is CancellationError { try? removeItemIfExists(IosSpeechModelPaths.stagingDirectory) @@ -461,8 +642,7 @@ final class IosSpeechModelStore: ObservableObject { nonisolated private static func manifestFiles() -> [IosSpeechModelManifestFile] { let manifest = EmbeddedSpeechModel.shared.manifest - return manifest.files.compactMap { item in - guard let file = item as? SpeechModelFile else { return nil } + return manifest.files.map { file in return IosSpeechModelManifestFile( name: file.name, sizeBytes: file.sizeBytes, @@ -484,6 +664,12 @@ final class IosSpeechModelStore: ObservableObject { } } + private struct PreparationResult { + let directory: URL? + let invalidInstallation: Bool + let message: String? + } + private struct IosSpeechModelManifestFile { let name: String let sizeBytes: Int64 diff --git a/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift b/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift new file mode 100644 index 0000000..5b710de --- /dev/null +++ b/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift @@ -0,0 +1,144 @@ +import Foundation +import Shared +import XCTest +@testable import VoiceInbox + +final class DeferredSpeechModelLoadingTests: XCTestCase { + func testLightweightInspectionDistinguishesMissingLegacyVerifiedAndKnownInvalidWithoutReadingPayloads() throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let model = root.appendingPathComponent("SpeechModel", isDirectory: true) + let receipt = root.appendingPathComponent("SpeechModel.receipt") + let invalid = root.appendingPathComponent("SpeechModel.invalid") + defer { try? FileManager.default.removeItem(at: root) } + + let missing = IosSpeechModelStore.inspectLightweight( + directory: model, + receiptFile: receipt, + invalidFile: invalid, + requiredFileNames: ["large-model.onnx"] + ) + XCTAssertEqual(missing.installationState, .missing) + + try FileManager.default.createDirectory(at: model, withIntermediateDirectories: true) + // Deliberately invalid payload contents prove the startup probe only checks presence/readability. + try Data("not an ONNX model".utf8).write(to: model.appendingPathComponent("large-model.onnx")) + let legacy = IosSpeechModelStore.inspectLightweight( + directory: model, + receiptFile: receipt, + invalidFile: invalid, + requiredFileNames: ["large-model.onnx"] + ) + XCTAssertEqual(legacy.installationState, .installedLegacy) + XCTAssertTrue(legacy.isReady) + + try EmbeddedSpeechModel.shared.manifest.version.write(to: receipt, atomically: true, encoding: .utf8) + let verified = IosSpeechModelStore.inspectLightweight( + directory: model, + receiptFile: receipt, + invalidFile: invalid, + requiredFileNames: ["large-model.onnx"] + ) + XCTAssertEqual(verified.installationState, .installedVerified) + + try "checksum mismatch".write(to: invalid, atomically: true, encoding: .utf8) + let knownInvalid = IosSpeechModelStore.inspectLightweight( + directory: model, + receiptFile: receipt, + invalidFile: invalid, + requiredFileNames: ["large-model.onnx"] + ) + XCTAssertEqual(knownInvalid.installationState, .invalid) + XCTAssertFalse(knownInvalid.isReady) + } + + @MainActor + func testLegacyPreparationWritesReceiptAndConcurrentAndLaterCallsReuseSingleLoad() async throws { + let root = FileManager.default.temporaryDirectory + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let receipt = root.appendingPathComponent("SpeechModel.receipt") + defer { try? FileManager.default.removeItem(at: root) } + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + let loads = LockedCounter() + + let store = IosSpeechModelStore( + directory: root, + inspectInstallation: { directory in + IosSpeechModelStatus( + directory: directory, + installationState: FileManager.default.fileExists(atPath: receipt.path) + ? .installedVerified + : .installedLegacy, + missingFiles: [] + ) + }, + validateInstallation: { _ in [] }, + prepareNative: { _ in + loads.increment() + Thread.sleep(forTimeInterval: 0.05) + return true + }, + nativeError: { nil }, + recordVerified: { + try? EmbeddedSpeechModel.shared.manifest.version.write( + to: receipt, + atomically: true, + encoding: .utf8 + ) + }, + recordInvalid: { _ in }, + resetNative: {} + ) + + async let first = store.prepareForTranscription() + async let second = store.prepareForTranscription() + let results = await [first, second] + + XCTAssertEqual(results.compactMap { $0 }.count, 2) + XCTAssertEqual(loads.value, 1) + XCTAssertEqual(store.status.installationState, .installedVerified) + XCTAssertTrue(FileManager.default.fileExists(atPath: receipt.path)) + + _ = await store.prepareForTranscription() + XCTAssertEqual(loads.value, 1) + + store.invalidateRuntimeAfterReplacement() + XCTAssertEqual(store.runtimeState, .unloaded) + _ = await store.prepareForTranscription() + XCTAssertEqual(loads.value, 2) + } + + @MainActor + func testPreparationFailureDoesNotClaimAudioAndDeferredHandoffDoesNotPrepareEarly() async { + let prepares = LockedCounter() + let claims = LockedCounter() + + XCTAssertEqual(prepares.value, 0) + XCTAssertEqual(claims.value, 0) + + let handedOff = await IosTranscriptionPreparationGate.prepareAndClaim( + prepare: { + prepares.increment() + return false + }, + claim: { claims.increment() } + ) + + XCTAssertFalse(handedOff) + XCTAssertEqual(prepares.value, 1) + XCTAssertEqual(claims.value, 0) + } +} + +private final class LockedCounter: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + + var value: Int { + lock.withLock { count } + } + + func increment() { + lock.withLock { count += 1 } + } +} diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/MainScreenStateController.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/MainScreenStateController.kt index 752cf3a..fd6c9b8 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/MainScreenStateController.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/MainScreenStateController.kt @@ -13,10 +13,10 @@ data class MainScreenRowInput( data class MainScreenInput( val modelMessage: String, - val modelLoading: Boolean, + val modelInstallationState: SpeechModelInstallationState, + val modelRuntimeState: SpeechModelRuntimeState, val modelDownloadAvailable: Boolean, val modelDownloadProgress: Int?, - val modelReady: Boolean, val outputSelected: Boolean, val folderSelected: Boolean, val pendingCount: Int, @@ -77,7 +77,7 @@ object MainScreenStateController { fun state(input: MainScreenInput): MainScreenState { val busy = input.folderChecking || input.folderScanQueued || input.scanning val controls = TranscriptionUiRules.catalogControls( - modelReady = input.modelReady, + modelInstallationState = input.modelInstallationState, outputSelected = input.outputSelected, folderSelected = input.folderSelected, pendingCount = input.pendingCount, @@ -132,10 +132,10 @@ object MainScreenStateController { private fun MainScreenInput.toStatusInput(): StatusProgressInput = StatusProgressInput( modelMessage = modelMessage, - modelLoading = modelLoading, + modelInstallationState = modelInstallationState, + modelRuntimeState = modelRuntimeState, modelDownloadAvailable = modelDownloadAvailable, modelDownloadProgress = modelDownloadProgress, - modelReady = modelReady, outputSelected = outputSelected, folderSelected = folderSelected, pendingCount = pendingCount, diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SpeechModelLifecycle.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SpeechModelLifecycle.kt new file mode 100644 index 0000000..f8888cd --- /dev/null +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SpeechModelLifecycle.kt @@ -0,0 +1,21 @@ +package me.maxistar.voiceinbox.core + +enum class SpeechModelInstallationState { + NOT_INSTALLED, + INSTALLING, + INSTALLED, + INVALID, +} + +enum class SpeechModelRuntimeState { + UNLOADED, + LOADING, + LOADED, + FAILED, +} + +val SpeechModelInstallationState.isAvailable: Boolean + get() = this == SpeechModelInstallationState.INSTALLED + +val SpeechModelRuntimeState.isPreparing: Boolean + get() = this == SpeechModelRuntimeState.LOADING diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt index 1b587fc..472be6d 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRules.kt @@ -16,7 +16,7 @@ data class StartupProcessingInput( val failedCount: Int = 0, val folderReady: Boolean = false, val outputReady: Boolean = false, - val modelReady: Boolean = false, + val modelInstallationState: SpeechModelInstallationState = SpeechModelInstallationState.NOT_INSTALLED, val transcriptionStateKnown: Boolean = false, val transcriptionActive: Boolean = false, ) @@ -55,7 +55,7 @@ object StartupProcessingRules { if (input.policy == StartupProcessingPolicy.LEAVE_QUEUED) { return StartupProcessingDecision.Finish(StartupProcessingFinishReason.LEAVE_QUEUED) } - if (!input.folderReady || !input.outputReady || !input.modelReady || input.transcriptionActive) { + if (!input.folderReady || !input.outputReady || !input.modelInstallationState.isAvailable || input.transcriptionActive) { return StartupProcessingDecision.Wait } return when (input.policy) { diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRules.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRules.kt index 3366394..988989c 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRules.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRules.kt @@ -12,10 +12,10 @@ data class CatalogControlState( data class StatusProgressInput( val modelMessage: String, - val modelLoading: Boolean, + val modelInstallationState: SpeechModelInstallationState, + val modelRuntimeState: SpeechModelRuntimeState, val modelDownloadAvailable: Boolean, val modelDownloadProgress: Int?, - val modelReady: Boolean, val outputSelected: Boolean, val folderSelected: Boolean, val pendingCount: Int, @@ -77,7 +77,7 @@ object TranscriptionUiRules { } fun catalogControls( - modelReady: Boolean, + modelInstallationState: SpeechModelInstallationState, outputSelected: Boolean, folderSelected: Boolean, pendingCount: Int, @@ -87,13 +87,14 @@ object TranscriptionUiRules { val workReady = transcriptionState == TranscriptionObservationState.IDLE || transcriptionState == TranscriptionObservationState.FINISHED val idle = workReady && !scanning - val prerequisites = modelReady && outputSelected && folderSelected && idle + val modelAvailable = modelInstallationState.isAvailable + val prerequisites = modelAvailable && outputSelected && folderSelected && idle return CatalogControlState( - outputEnabled = modelReady && idle, - folderEnabled = modelReady && idle, + outputEnabled = modelAvailable && idle, + folderEnabled = modelAvailable && idle, outputSetupVisible = !outputSelected, folderSetupVisible = !folderSelected, - refreshEnabled = modelReady && folderSelected && idle, + refreshEnabled = modelAvailable && folderSelected && idle, transcribeAllEnabled = prerequisites && pendingCount > 0, retryEnabled = prerequisites, ) @@ -126,7 +127,7 @@ object TranscriptionUiRules { fun statusProgressBlock(input: StatusProgressInput): StatusProgressBlockState = when { input.transcriptionState == TranscriptionObservationState.ACTIVE -> activeTranscription(input) - input.modelLoading -> StatusProgressBlockState( + input.modelInstallationState == SpeechModelInstallationState.INSTALLING -> StatusProgressBlockState( title = input.modelMessage, progressVisible = true, progressIndeterminate = input.modelDownloadProgress == null, @@ -142,7 +143,7 @@ object TranscriptionUiRules { progressVisible = true, progressIndeterminate = true, ) - !input.modelReady -> StatusProgressBlockState( + !input.modelInstallationState.isAvailable -> StatusProgressBlockState( title = input.modelMessage, downloadVisible = input.modelDownloadAvailable, ) diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/MainScreenStateControllerTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/MainScreenStateControllerTest.kt index d1c1d97..9ac38a3 100644 --- a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/MainScreenStateControllerTest.kt +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/MainScreenStateControllerTest.kt @@ -48,6 +48,29 @@ class MainScreenStateControllerTest { ) } + @Test + fun installedModelDoesNotRequireLoadedRuntimeForActions() { + val unloaded = state( + input( + outputSelected = true, + folderSelected = true, + pendingCount = 1, + modelRuntimeState = SpeechModelRuntimeState.UNLOADED, + ), + ) + val retryableFailure = state( + input( + outputSelected = true, + folderSelected = true, + pendingCount = 1, + modelRuntimeState = SpeechModelRuntimeState.FAILED, + ), + ) + + assertTrue(unloaded.controls.transcribeAllEnabled) + assertTrue(retryableFailure.controls.transcribeAllEnabled) + } + @Test fun controlsAreDisabledDuringActiveWorkOrQueuedScan() { assertFalse( @@ -256,6 +279,7 @@ class MainScreenStateControllerTest { modelDownloadAvailable: Boolean = false, modelDownloadProgress: Int? = null, modelReady: Boolean = true, + modelRuntimeState: SpeechModelRuntimeState = SpeechModelRuntimeState.UNLOADED, outputSelected: Boolean = false, folderSelected: Boolean = false, pendingCount: Int = 0, @@ -281,10 +305,14 @@ class MainScreenStateControllerTest { rows: List = emptyList(), ) = MainScreenInput( modelMessage = modelMessage, - modelLoading = modelLoading, + modelInstallationState = when { + modelLoading -> SpeechModelInstallationState.INSTALLING + modelReady -> SpeechModelInstallationState.INSTALLED + else -> SpeechModelInstallationState.NOT_INSTALLED + }, + modelRuntimeState = modelRuntimeState, modelDownloadAvailable = modelDownloadAvailable, modelDownloadProgress = modelDownloadProgress, - modelReady = modelReady, outputSelected = outputSelected, folderSelected = folderSelected, pendingCount = pendingCount, diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt index e25534e..cbe133b 100644 --- a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/StartupProcessingRulesTest.kt @@ -36,7 +36,7 @@ class StartupProcessingRulesTest { ready.copy(catalogReady = false), ready.copy(folderReady = false), ready.copy(outputReady = false), - ready.copy(modelReady = false), + ready.copy(modelInstallationState = SpeechModelInstallationState.NOT_INSTALLED), ready.copy(transcriptionStateKnown = false), ) @@ -102,7 +102,7 @@ class StartupProcessingRulesTest { failedCount = failedCount, folderReady = true, outputReady = true, - modelReady = true, + modelInstallationState = SpeechModelInstallationState.INSTALLED, transcriptionStateKnown = true, transcriptionActive = transcriptionActive, ) diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRulesTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRulesTest.kt index 8991c3e..89d6a59 100644 --- a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRulesTest.kt +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TranscriptionUiRulesTest.kt @@ -44,7 +44,7 @@ class TranscriptionUiRulesTest { assertEquals( CatalogControlState(false, false, false, false, false, false, false), TranscriptionUiRules.catalogControls( - modelReady = true, + modelInstallationState = SpeechModelInstallationState.INSTALLED, outputSelected = true, folderSelected = true, pendingCount = 2, @@ -55,7 +55,7 @@ class TranscriptionUiRulesTest { assertEquals( CatalogControlState(false, false, false, false, false, false, false), TranscriptionUiRules.catalogControls( - modelReady = true, + modelInstallationState = SpeechModelInstallationState.INSTALLED, outputSelected = true, folderSelected = true, pendingCount = 2, @@ -70,7 +70,7 @@ class TranscriptionUiRulesTest { assertEquals( CatalogControlState(false, false, false, false, false, false, false), TranscriptionUiRules.catalogControls( - modelReady = true, + modelInstallationState = SpeechModelInstallationState.INSTALLED, outputSelected = true, folderSelected = true, pendingCount = 2, @@ -432,7 +432,7 @@ class TranscriptionUiRulesTest { pending: Int = 0, active: Boolean = false, ) = TranscriptionUiRules.catalogControls( - modelReady, + if (modelReady) SpeechModelInstallationState.INSTALLED else SpeechModelInstallationState.NOT_INSTALLED, output, folder, pending, @@ -466,10 +466,14 @@ class TranscriptionUiRulesTest { ) = TranscriptionUiRules.statusProgressBlock( StatusProgressInput( modelMessage = modelMessage, - modelLoading = modelLoading, + modelInstallationState = when { + modelLoading -> SpeechModelInstallationState.INSTALLING + modelReady -> SpeechModelInstallationState.INSTALLED + else -> SpeechModelInstallationState.NOT_INSTALLED + }, + modelRuntimeState = SpeechModelRuntimeState.UNLOADED, modelDownloadAvailable = modelDownloadAvailable, modelDownloadProgress = modelDownloadProgress, - modelReady = modelReady, outputSelected = output, folderSelected = folder, pendingCount = pending, diff --git a/src/engine.rs b/src/engine.rs index 69cde49..4a8e044 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -33,16 +33,77 @@ pub fn is_engine_loaded() -> bool { } pub fn configure_model_directory(path: PathBuf) { + let path = std::fs::canonicalize(&path).unwrap_or(path); + let unchanged = MODEL_DIRECTORY.lock().unwrap().as_ref() == Some(&path); + if unchanged { + return; + } + invalidate_loaded_model(); *MODEL_DIRECTORY.lock().unwrap() = Some(path); +} + +pub fn invalidate_loaded_model() { *GLOBAL_ENGINE.lock().unwrap() = None; *LOAD_STATE.0.lock().unwrap() = LoadState::Idle; + LOAD_STATE.1.notify_all(); } pub fn ensure_loaded_without_callback() -> Result<(), String> { + ensure_loaded_with_status(|_| {}) +} + +fn ensure_loaded_with_status(mut status: impl FnMut(&str)) -> Result<(), String> { + if is_engine_loaded() { + status("Ready"); + return Ok(()); + } + + let (lock, cvar) = &*LOAD_STATE; + let mut state = lock.lock().unwrap(); if is_engine_loaded() { + status("Ready"); return Ok(()); } + if *state == LoadState::Loading { + status("Waiting for model..."); + while *state == LoadState::Loading { + state = cvar.wait(state).unwrap(); + } + return match &*state { + LoadState::Done if is_engine_loaded() => { + status("Ready"); + Ok(()) + } + LoadState::Failed(message) => { + status(&format!("Error: {message}")); + Err(message.clone()) + } + _ => Err("Model loading was interrupted".to_string()), + }; + } + + *state = LoadState::Loading; + drop(state); + status("Loading model..."); + + let result = load_configured_engine(); + let mut state = lock.lock().unwrap(); + match &result { + Ok(()) => { + *state = LoadState::Done; + status("Ready"); + } + Err(message) => { + *state = LoadState::Failed(message.clone()); + status(&format!("Error: {message}")); + } + } + cvar.notify_all(); + result +} + +fn load_configured_engine() -> Result<(), String> { let path = MODEL_DIRECTORY .lock() .unwrap() @@ -56,7 +117,6 @@ pub fn ensure_loaded_without_callback() -> Result<(), String> { ) .map_err(|error| format!("Model error: {error}"))?; *GLOBAL_ENGINE.lock().unwrap() = Some(Arc::new(Mutex::new(engine))); - *LOAD_STATE.0.lock().unwrap() = LoadState::Done; Ok(()) } @@ -74,55 +134,7 @@ fn notify_status(env: &mut JNIEnv, obj: &JObject, msg: &str) { #[cfg(target_os = "android")] pub fn ensure_loaded(env: &mut JNIEnv, context: &JObject) -> Result<(), String> { - if is_engine_loaded() { - notify_status(env, context, "Ready"); - return Ok(()); - } - - let (lock, cvar) = &*LOAD_STATE; - let mut state = lock.lock().unwrap(); - - if is_engine_loaded() { - notify_status(env, context, "Ready"); - return Ok(()); - } - - match &*state { - LoadState::Loading => { - notify_status(env, context, "Waiting for model..."); - while *state == LoadState::Loading { - state = cvar.wait(state).unwrap(); - } - drop(state); - - if is_engine_loaded() { - notify_status(env, context, "Ready"); - Ok(()) - } else { - let msg = "Model failed to load".to_string(); - notify_status(env, context, &format!("Error: {}", msg)); - Err(msg) - } - } - LoadState::Done => { - notify_status(env, context, "Ready"); - Ok(()) - } - LoadState::Idle | LoadState::Failed(_) => { - *state = LoadState::Loading; - drop(state); - - let result = do_load(env, context); - - let mut state = lock.lock().unwrap(); - match &result { - Ok(()) => *state = LoadState::Done, - Err(msg) => *state = LoadState::Failed(msg.clone()), - } - cvar.notify_all(); - result - } - } + ensure_loaded_with_status(|message| notify_status(env, context, message)) } #[cfg(target_os = "android")] @@ -130,98 +142,44 @@ pub fn ensure_loaded_from_thread( jvm: &Arc, target_ref: &GlobalRef, ) -> Result<(), String> { - if is_engine_loaded() { + ensure_loaded_with_status(|message| { if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), "Ready"); + notify_status(&mut env, target_ref.as_obj(), message); } - return Ok(()); - } + }) +} - let (lock, cvar) = &*LOAD_STATE; - let mut state = lock.lock().unwrap(); +#[cfg(test)] +mod tests { + use super::*; - if is_engine_loaded() { - if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), "Ready"); - } - return Ok(()); - } + static TEST_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); - match &*state { - LoadState::Loading => { - if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), "Waiting for model..."); - } - while *state == LoadState::Loading { - state = cvar.wait(state).unwrap(); - } - drop(state); + #[test] + fn configuring_same_directory_preserves_load_state() { + let _guard = TEST_LOCK.lock().unwrap(); + invalidate_loaded_model(); + configure_model_directory(PathBuf::from("test-model")); + *LOAD_STATE.0.lock().unwrap() = LoadState::Done; - if is_engine_loaded() { - if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), "Ready"); - } - Ok(()) - } else { - let msg = "Model failed to load".to_string(); - if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), &format!("Error: {}", msg)); - } - Err(msg) - } - } - LoadState::Done => { - if let Ok(mut env) = jvm.attach_current_thread() { - notify_status(&mut env, target_ref.as_obj(), "Ready"); - } - Ok(()) - } - LoadState::Idle | LoadState::Failed(_) => { - *state = LoadState::Loading; - drop(state); - - let result = if let Ok(mut env) = jvm.attach_current_thread() { - let obj = target_ref.as_obj(); - do_load(&mut env, obj) - } else { - Err("Failed to attach JNI thread".to_string()) - }; - - let mut state = lock.lock().unwrap(); - match &result { - Ok(()) => *state = LoadState::Done, - Err(msg) => *state = LoadState::Failed(msg.clone()), - } - cvar.notify_all(); - result - } + configure_model_directory(PathBuf::from("test-model")); + + assert_eq!(*LOAD_STATE.0.lock().unwrap(), LoadState::Done); + invalidate_loaded_model(); } -} -#[cfg(target_os = "android")] -fn do_load(env: &mut JNIEnv, context: &JObject) -> Result<(), String> { - let path = MODEL_DIRECTORY - .lock() - .unwrap() - .clone() - .ok_or_else(|| "Model directory was not configured".to_string())?; + #[test] + fn changing_directory_and_explicit_invalidation_reset_state() { + let _guard = TEST_LOCK.lock().unwrap(); + invalidate_loaded_model(); + configure_model_directory(PathBuf::from("first-model")); + *LOAD_STATE.0.lock().unwrap() = LoadState::Done; - notify_status(env, context, "Loading model..."); - - let mut eng = ParakeetEngine::new(); - match eng.load_model_with_params( - &path, - transcribe_rs::engines::parakeet::ParakeetModelParams::int8(), - ) { - Ok(_) => { - *GLOBAL_ENGINE.lock().unwrap() = Some(Arc::new(Mutex::new(eng))); - notify_status(env, context, "Ready"); - Ok(()) - } - Err(e) => { - let msg = format!("Model error: {}", e); - notify_status(env, context, &format!("Error: {}", msg)); - Err(msg) - } + configure_model_directory(PathBuf::from("second-model")); + assert_eq!(*LOAD_STATE.0.lock().unwrap(), LoadState::Idle); + + *LOAD_STATE.0.lock().unwrap() = LoadState::Failed("failed".to_string()); + invalidate_loaded_model(); + assert_eq!(*LOAD_STATE.0.lock().unwrap(), LoadState::Idle); } } diff --git a/src/lib.rs b/src/lib.rs index 82ffa56..9725736 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,7 @@ use jni::sys::{jboolean, jstring, JNI_FALSE, JNI_TRUE}; use jni::JNIEnv; #[cfg(target_os = "android")] use std::path::PathBuf; -#[cfg(any(not(target_os = "ios"), all(target_os = "ios", feature = "ios-onnx")))] +#[cfg(any(target_os = "android", all(target_os = "ios", feature = "ios-onnx")))] use transcribe_rs::TranscriptionEngine; #[cfg(any(not(target_os = "ios"), all(target_os = "ios", feature = "ios-onnx")))] @@ -74,6 +74,15 @@ pub unsafe extern "system" fn Java_me_maxistar_voiceinbox_NativeTranscriptionBri } } +#[cfg(target_os = "android")] +#[no_mangle] +pub extern "system" fn Java_me_maxistar_voiceinbox_NativeTranscriptionBridge_reset( + _env: JNIEnv, + _class: JClass, +) { + engine::invalidate_loaded_model(); +} + #[cfg(target_os = "android")] #[no_mangle] pub unsafe extern "system" fn Java_me_maxistar_voiceinbox_NativeTranscriptionBridge_transcribeChunkJson( @@ -86,7 +95,10 @@ pub unsafe extern "system" fn Java_me_maxistar_voiceinbox_NativeTranscriptionBri Err(_) => return std::ptr::null_mut(), }; let mut buffer = vec![0.0f32; length]; - if env.get_float_array_region(&samples, 0, &mut buffer).is_err() { + if env + .get_float_array_region(&samples, 0, &mut buffer) + .is_err() + { return std::ptr::null_mut(); } @@ -226,6 +238,16 @@ pub unsafe extern "C" fn voiceinbox_transcription_initialize( true } +#[cfg(all(target_os = "ios", feature = "ios-onnx"))] +#[no_mangle] +pub extern "C" fn voiceinbox_transcription_reset() { + engine::invalidate_loaded_model(); +} + +#[cfg(all(target_os = "ios", not(feature = "ios-onnx")))] +#[no_mangle] +pub extern "C" fn voiceinbox_transcription_reset() {} + #[cfg(all(target_os = "ios", not(feature = "ios-onnx")))] #[no_mangle] pub unsafe extern "C" fn voiceinbox_transcription_initialize( From 4ee86e0884aaa914245461205a8a32245a1361ce Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Thu, 16 Jul 2026 07:50:42 +0200 Subject: [PATCH 05/10] add list style ui --- iosApp/VoiceInbox/ContentView.swift | 546 ++++++++---------- iosApp/VoiceInbox/IosAudioImportStore.swift | 47 +- .../VoiceInbox/IosMainScreenShellState.swift | 335 +++++------ ...IosSingleFileTranscriptionController.swift | 28 +- iosApp/VoiceInbox/IosSpeechModelStore.swift | 4 + .../DeferredSpeechModelLoadingTests.swift | 113 ++++ .../maxistar/voiceinbox/core/AudioCatalog.kt | 4 + .../core/BatchTranscriptionUseCase.kt | 9 +- .../core/SqlDelightAudioCatalogRepository.kt | 6 +- .../core/TaskListPresentationController.kt | 431 ++++++++++++++ .../me/maxistar/voiceinbox/db/AudioFiles.sq | 2 +- .../core/BatchTranscriptionUseCaseTest.kt | 9 + .../TaskListPresentationControllerTest.kt | 199 +++++++ 13 files changed, 1236 insertions(+), 497 deletions(-) create mode 100644 shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationController.kt create mode 100644 shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationControllerTest.kt diff --git a/iosApp/VoiceInbox/ContentView.swift b/iosApp/VoiceInbox/ContentView.swift index 9ccecd2..e1d0826 100644 --- a/iosApp/VoiceInbox/ContentView.swift +++ b/iosApp/VoiceInbox/ContentView.swift @@ -25,329 +25,100 @@ struct ContentView: View { let transcriptionBackendConfigured = transcriber.backendConfigured let speechModelReady = speechModelStore.isReady let outputReady = outputStore.isReady - let storageSetupComplete = outputReady && !importStore.inboxFolderStatus.needsSelection let transcriptionReady = transcriptionBackendConfigured && speechModelReady && !speechModelStore.isBusy && outputReady - let modelStatusMessage = iOSModelStatusMessage( - transcriptionBackendConfigured: transcriptionBackendConfigured, - speechModelReady: speechModelReady - ) let modelDownloadAvailable = !speechModelReady && !speechModelStore.isBusy - let modelSetupVisible = !transcriptionBackendConfigured || !speechModelReady || speechModelStore.isBusy let screen = shellState.screen( selection: selectedTab, importedFiles: importStore.files, - runtimeReady: transcriptionBackendConfigured, - modelReady: speechModelReady, + modelStatus: speechModelStore.status, + modelMessage: speechModelStore.message, modelInstalling: speechModelStore.isInstalling, modelDownloadAvailable: modelDownloadAvailable, modelDownloadProgress: speechModelStore.downloadProgress?.percent, - modelMessage: modelStatusMessage, - outputReady: outputReady, + modelCanCancel: speechModelStore.canCancelDownload, + outputStatus: outputStore.status, + folderStatus: importStore.inboxFolderStatus, + folderScanning: importStore.isScanningFolder, activePreviewEntryId: previewPlayer.playingFileId, previewState: previewPlayer.playingFileId == nil ? PreviewPlaybackState.idle : PreviewPlaybackState.playing, - transcription: transcriber.state + transcription: transcriber.state, + preparationOwnerEntryId: transcriber.preparationOwnerFileId, + prerequisiteError: transcriber.prerequisiteError, + actionsEnabled: transcriptionReady && !transcriber.isActive && !importStore.isScanningFolder ) NavigationStack { List { - - Section("Status") { - VStack(alignment: .leading, spacing: 8) { - Text(screen.state.status.title) - .font(.headline) - if let detail = screen.state.status.detail { - Text(detail) - .foregroundStyle(.secondary) - } - if let meta = screen.state.status.meta { - Text(meta) - .font(.footnote) - .foregroundStyle(.secondary) - } - } - + Section { Picker("Catalog", selection: $selectedTab) { ForEach(IosShellCatalogSelection.allCases) { tab in Text(tab.title).tag(tab) } } .pickerStyle(.segmented) - - if screen.state.list.transcribeAllVisible { - Button { - guard let outputDocument = outputStore.currentDocument() else { - outputStore.refreshAccess() - return - } - previewPlayer.stop() - transcriber.transcribeAll( - modelDirectory: speechModelStore.modelDirectory, - modelStore: speechModelStore, - outputDocument: outputDocument, - store: importStore, - onFinished: { - selectedTab = .processed - } - ) - } label: { - Label("Transcribe All", systemImage: "text.badge.checkmark") - } - .disabled(!screen.state.list.transcribeAllEnabled) - } - - Button { - showingImporter = true - } label: { - Label("Import Audio Files", systemImage: "square.and.arrow.down") - } - - if !outputReady { - VStack(alignment: .leading, spacing: 6) { - Text(outputStore.status.title) - .font(.headline) - Text(outputStore.status.message) - .font(.footnote) - .foregroundStyle(.secondary) - } - - Button { - showingOutputPicker = true - } label: { - Label("Select Output File", systemImage: "doc.badge.plus") - } - } - - if importStore.inboxFolderStatus.needsSelection { - VStack(alignment: .leading, spacing: 6) { - Text(importStore.inboxFolderStatus.title) - .font(.headline) - if let message = importStore.inboxFolderStatus.message { - Text(message) - .font(.footnote) - .foregroundStyle(.secondary) - } - } - - Button { - showingInboxFolderPicker = true - } label: { - Label("Select Audio Folder", systemImage: "folder") - } - .disabled(importStore.isScanningFolder) - } - - if !importStore.inboxFolderStatus.needsSelection { - Button { - importStore.refreshInboxFolder() - selectedTab = .new - } label: { - Label("Refresh Audio Folder", systemImage: "arrow.clockwise") - } - .disabled(importStore.isScanningFolder) - } - - if importStore.isScanningFolder { - ProgressView() - } + .accessibilityIdentifier("task-list-filter") } Section { - if screen.state.list.emptyVisible { + if let emptyMessage = screen.state.emptyMessage { VStack(alignment: .leading, spacing: 8) { - Text(screen.state.list.emptyMessage) + Text(emptyMessage) .foregroundStyle(.secondary) - Button { - showingInboxFolderPicker = true - } label: { - Label("Select Audio Folder", systemImage: "folder") - } - Button { - showingImporter = true - } label: { - Label("Import Audio Files", systemImage: "square.and.arrow.down") - } - } - } else { - ForEach(screen.rows) { row in - VStack(alignment: .leading, spacing: 8) { - HStack(alignment: .top) { - VStack(alignment: .leading, spacing: 4) { - Text(row.title) - .font(.headline) - Text(row.subtitle) - .font(.subheadline) - .foregroundStyle(.secondary) - } - - Spacer() - - Text(row.badge) - .font(.caption) - .padding(.horizontal, 8) - .padding(.vertical, 4) - .background(.thinMaterial) - .clipShape(Capsule()) - } - - HStack { - if let importedFile = importedFile(for: row) { - Button { - previewPlayer.toggle( - fileId: importedFile.id, - url: importStore.localURL(for: importedFile) - ) - } label: { - Label( - previewPlayer.playingFileId == importedFile.id ? "Stop" : row.state.preview.label, - systemImage: previewPlayer.playingFileId == importedFile.id ? "stop.fill" : "play.fill" - ) - } - .disabled( - transcriber.isActive || - !row.state.preview.enabled - ) - - if importedFile.status == .pending { - Button { - guard let outputDocument = outputStore.currentDocument() else { - outputStore.refreshAccess() - return - } - transcriber.transcribe( - file: importedFile, - localURL: importStore.localURL(for: importedFile), - modelDirectory: speechModelStore.modelDirectory, - modelStore: speechModelStore, - outputDocument: outputDocument, - store: importStore, - onSuccess: { transcript in - shownTranscript = transcript - selectedTab = .processed - } - ) - } label: { - Label("Transcribe", systemImage: "text.badge.checkmark") - } - .disabled( - transcriber.isActive || - !transcriptionReady - ) - } - - if importedFile.status == .failed { - Button { - guard let outputDocument = outputStore.currentDocument() else { - outputStore.refreshAccess() - return - } - transcriber.retry( - file: importedFile, - localURL: importStore.localURL(for: importedFile), - modelDirectory: speechModelStore.modelDirectory, - modelStore: speechModelStore, - outputDocument: outputDocument, - store: importStore, - onSuccess: { transcript in - shownTranscript = transcript - selectedTab = .processed - } - ) - } label: { - Label("Retry", systemImage: "arrow.clockwise") - } - .disabled( - transcriber.isActive || - !transcriptionReady - ) - } - } - - if row.state.showTextVisible { - Button("Show Text") { - shownTranscript = row.transcriptText - } - } + ForEach(Array(screen.state.emptyActions.enumerated()), id: \.offset) { _, action in + Button(action.label) { + perform(action: action, task: nil, screen: screen) } - .buttonStyle(.bordered) - .font(.caption) + .disabled(!action.enabled) } - .padding(.vertical, 4) } - } - } header: { - if !storageSetupComplete { - Text(selectedTab.title) - } - } - - if let importMessage = importStore.importMessage { - Section("Import result") { - Text(importMessage) - .foregroundStyle(.secondary) - } - } - - if let playbackError = previewPlayer.errorMessage { - Section("Playback") { - Text(playbackError) - .foregroundStyle(.secondary) - Button("Dismiss") { - previewPlayer.clearError() - } - } - } - - if modelSetupVisible { - Section("Transcription Setup") { - if !transcriptionBackendConfigured { - Text(IosSingleFileTranscriptionController.backendUnavailableMessage) - .foregroundStyle(.secondary) - } - VStack(alignment: .leading, spacing: 6) { - Text(speechModelStore.status.summary) - if speechModelStore.isInstalling { - if let progress = speechModelStore.downloadProgress { - ProgressView(value: Double(progress.percent), total: 100) - Text("\(progress.percent)% • \(formatBytes(progress.bytesDownloaded)) of \(formatBytes(progress.totalBytes))") - .font(.footnote) - .foregroundStyle(.secondary) - } else { - ProgressView() - } - } - if let detail = speechModelStore.status.detail { - Text(detail) - .font(.footnote) - .foregroundStyle(.secondary) - } - if let modelMessage = speechModelStore.message { - Text(modelMessage) - .font(.footnote) - .foregroundStyle(.secondary) + } else { + ForEach(screen.state.tasks.filter { $0 is SetupTaskPresentation }, id: \.stableId) { task in + TaskListRow(task: task) { action in + perform(action: action, task: task, screen: screen) } + .id(task.stableId) + .accessibilityIdentifier("task-row-\(task.stableId)") } - if modelDownloadAvailable { + if screen.state.batchAction.visible { Button { - speechModelStore.downloadModel() + transcribeAll() } label: { - Label("Download Speech Model", systemImage: "arrow.down.circle") + Label( + "Transcribe All (\(screen.state.batchAction.eligibleCount))", + systemImage: "text.badge.checkmark" + ) } - .buttonStyle(.borderedProminent) + .disabled(!screen.state.batchAction.enabled) + .accessibilityIdentifier("transcribe-all") } - if !speechModelReady { - Button { - showingModelImporter = true - } label: { - Label("Install Speech Model Manually", systemImage: "square.and.arrow.down") + ForEach(screen.state.tasks.filter { $0 is AudioTaskPresentation }, id: \.stableId) { task in + TaskListRow(task: task) { action in + perform(action: action, task: task, screen: screen) } - .disabled(speechModelStore.isBusy) + .id(task.stableId) + .accessibilityIdentifier("task-row-\(task.stableId)") } - } } + Section { + Button { + showingImporter = true + } label: { + Label("Import Audio Files", systemImage: "square.and.arrow.down") + } + if !importStore.inboxFolderStatus.needsSelection { + Button { + importStore.refreshInboxFolder() + selectedTab = .new + } label: { + Label("Refresh Audio Folder", systemImage: "arrow.clockwise") + } + .disabled(importStore.isScanningFolder) + } + } } .navigationTitle("Voice Inbox") .toolbar { @@ -443,6 +214,13 @@ struct ContentView: View { } ) } + .alert(item: globalMessageBinding) { message in + Alert( + title: Text(message.title), + message: Text(message.text), + dismissButton: .default(Text("OK")) + ) + } } } @@ -457,9 +235,111 @@ struct ContentView: View { ) } - private func importedFile(for row: IosShellAudioRow) -> IosImportedAudioFile? { - guard row.imported else { return nil } - return importStore.files.first { $0.id == row.id } + private var globalMessageBinding: Binding { + Binding( + get: { + if let error = previewPlayer.errorMessage { + return IosGlobalMessage(title: "Playback", text: error) + } + if let message = importStore.importMessage { + return IosGlobalMessage(title: "Voice Inbox", text: message) + } + return nil + }, + set: { value in + guard value == nil else { return } + previewPlayer.clearError() + importStore.importMessage = nil + } + ) + } + + private func perform( + action: TaskActionPresentation, + task: TaskPresentation?, + screen: IosTaskListScreen + ) { + guard action.enabled, let route = IosTaskActionRouter.route(action.kind) else { return } + switch route { + case .modelDownload: + speechModelStore.downloadModel() + case .modelImport: + showingModelImporter = true + case .modelCancel: + speechModelStore.cancelDownload() + case .outputSelection: + showingOutputPicker = true + case .folderSelection: + showingInboxFolderPicker = true + case .folderRefresh: + importStore.refreshInboxFolder() + selectedTab = .new + case .audioImport: + showingImporter = true + case .transcribe, .retry, .play, .stop, .showText: + guard let audioTask = task as? AudioTaskPresentation, + let file = screen.filesById[audioTask.entryId] else { return } + performAudio(action: action.kind, file: file) + } + } + + private func performAudio(action: TaskActionKind, file: IosImportedAudioFile) { + switch action { + case .play: + previewPlayer.toggle(fileId: file.id, url: importStore.localURL(for: file)) + case .stop: + previewPlayer.stop() + case .showText: + shownTranscript = file.transcriptText + case .transcribe, .retryTranscription: + guard let outputDocument = outputStore.currentDocument() else { + outputStore.refreshAccess() + return + } + previewPlayer.stop() + let onSuccess: (String) -> Void = { transcript in + shownTranscript = transcript + selectedTab = .processed + } + if action == .retryTranscription { + transcriber.retry( + file: file, + localURL: importStore.localURL(for: file), + modelDirectory: speechModelStore.modelDirectory, + modelStore: speechModelStore, + outputDocument: outputDocument, + store: importStore, + onSuccess: onSuccess + ) + } else { + transcriber.transcribe( + file: file, + localURL: importStore.localURL(for: file), + modelDirectory: speechModelStore.modelDirectory, + modelStore: speechModelStore, + outputDocument: outputDocument, + store: importStore, + onSuccess: onSuccess + ) + } + default: + break + } + } + + private func transcribeAll() { + guard let outputDocument = outputStore.currentDocument() else { + outputStore.refreshAccess() + return + } + previewPlayer.stop() + transcriber.transcribeAll( + modelDirectory: speechModelStore.modelDirectory, + modelStore: speechModelStore, + outputDocument: outputDocument, + store: importStore, + onFinished: { selectedTab = .processed } + ) } private func refreshStartupSources() { @@ -530,31 +410,87 @@ struct ContentView: View { ) } - private func iOSModelStatusMessage( - transcriptionBackendConfigured: Bool, - speechModelReady: Bool - ) -> String { - if !transcriptionBackendConfigured { - return IosSingleFileTranscriptionController.backendUnavailableMessage - } - if let progress = speechModelStore.downloadProgress { - return progress.message +} + +private struct TaskListRow: View { + let task: TaskPresentation + let onAction: (TaskActionPresentation) -> Void + + var body: some View { + VStack(alignment: .leading, spacing: 10) { + HStack(alignment: .top) { + VStack(alignment: .leading, spacing: 4) { + Text(task.title) + .font(.headline) + if let detail = task.detail, !detail.isEmpty { + Text(detail) + .font(.subheadline) + .foregroundStyle(.secondary) + } + } + Spacer() + Text(task.badge) + .font(.caption) + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(.thinMaterial) + .clipShape(Capsule()) + } + + if let error = task.errorMessage, !error.isEmpty { + Text(error) + .font(.footnote) + .foregroundStyle(.red) + } + + if let progress = task.progress { + VStack(alignment: .leading, spacing: 4) { + if let percent = progress.percent?.int32Value { + ProgressView(value: Double(percent), total: 100) + .accessibilityIdentifier("task-progress-\(task.stableId)") + } else { + ProgressView() + .accessibilityIdentifier("task-progress-\(task.stableId)") + } + Text(progressLabel(progress)) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + + if !task.actions.isEmpty { + HStack { + ForEach(Array(task.actions.enumerated()), id: \.offset) { _, action in + Button(action.label) { onAction(action) } + .disabled(!action.enabled) + .accessibilityIdentifier("task-action-\(task.stableId)-\(action.kind.name.lowercased())") + } + } + .buttonStyle(.bordered) + .font(.caption) + } } - if speechModelStore.isInstalling { - return "Installing speech model..." + .padding(.vertical, 4) + } + + private func progressLabel(_ progress: TaskProgressPresentation) -> String { + var parts = [progress.phase] + if let completed = progress.completedFiles?.int32Value, + let total = progress.totalFiles?.int32Value, + total > 0 { + parts.append("\(completed) / \(total) files") } - if !speechModelReady { - return speechModelStore.status.summary + if let failed = progress.failedFiles?.int32Value, failed > 0 { + parts.append("\(failed) failed") } - return "Ready" + return parts.joined(separator: " • ") } +} - private func formatBytes(_ bytes: Int64) -> String { - let formatter = ByteCountFormatter() - formatter.allowedUnits = [.useMB, .useGB] - formatter.countStyle = .file - return formatter.string(fromByteCount: bytes) - } +private struct IosGlobalMessage: Identifiable { + let id = UUID() + let title: String + let text: String } private struct IosDisplayedTranscript: Identifiable { diff --git a/iosApp/VoiceInbox/IosAudioImportStore.swift b/iosApp/VoiceInbox/IosAudioImportStore.swift index e7fa3e1..f1d6409 100644 --- a/iosApp/VoiceInbox/IosAudioImportStore.swift +++ b/iosApp/VoiceInbox/IosAudioImportStore.swift @@ -64,6 +64,7 @@ struct IosImportedAudioFile: Codable, Identifiable, Equatable { var transcriptText: String? var durationUs: Int64? var lastError: String? + var processedAt: Date? init( id: Int64, @@ -74,7 +75,8 @@ struct IosImportedAudioFile: Codable, Identifiable, Equatable { status: IosImportedAudioStatus = .pending, transcriptText: String? = nil, durationUs: Int64? = nil, - lastError: String? = nil + lastError: String? = nil, + processedAt: Date? = nil ) { self.id = id self.displayName = displayName @@ -85,6 +87,7 @@ struct IosImportedAudioFile: Codable, Identifiable, Equatable { self.transcriptText = transcriptText self.durationUs = durationUs self.lastError = lastError + self.processedAt = processedAt } init(from decoder: Decoder) throws { @@ -98,6 +101,7 @@ struct IosImportedAudioFile: Codable, Identifiable, Equatable { transcriptText = try container.decodeIfPresent(String.self, forKey: .transcriptText) durationUs = try container.decodeIfPresent(Int64.self, forKey: .durationUs) lastError = try container.decodeIfPresent(String.self, forKey: .lastError) + processedAt = try container.decodeIfPresent(Date.self, forKey: .processedAt) } var formattedSize: String { @@ -128,12 +132,33 @@ struct IosAudioImportSummary { if failed > 0 { parts.append("\(failed) failed") } return parts.isEmpty ? "No files imported" : parts.joined(separator: ", ") } + + var alertMessage: String? { + failed > 0 ? message : nil + } } struct IosInboxFolderStatus { let displayName: String? let message: String? let needsSelection: Bool + let hasError: Bool + + init(displayName: String?, message: String?, needsSelection: Bool) { + self.init( + displayName: displayName, + message: message, + needsSelection: needsSelection, + hasError: false + ) + } + + init(displayName: String?, message: String?, needsSelection: Bool, hasError: Bool) { + self.displayName = displayName + self.message = message + self.needsSelection = needsSelection + self.hasError = hasError + } var title: String { displayName ?? "No audio folder selected" @@ -171,7 +196,7 @@ final class IosAudioImportStore: ObservableObject { func importFiles(from urls: [URL]) { let summary = copySupportedFiles(from: urls) load() - importMessage = summary.message + importMessage = summary.alertMessage } @discardableResult @@ -190,7 +215,7 @@ final class IosAudioImportStore: ObservableObject { let summary = copySupportedSharedFiles(from: stagedFiles) load() - importMessage = summary.message + importMessage = summary.alertMessage return summary } @@ -235,10 +260,13 @@ final class IosAudioImportStore: ObservableObject { isScanningFolder = false inboxFolderStatus = IosInboxFolderStatus( displayName: folder.url.lastPathComponent, - message: summary.message, - needsSelection: false + message: summary.failed > 0 + ? "Audio folder scan failed. Check folder access and try again." + : summary.message, + needsSelection: false, + hasError: summary.failed > 0 ) - importMessage = summary.message + importMessage = nil } func localURL(for file: IosImportedAudioFile) -> URL { @@ -273,7 +301,7 @@ final class IosAudioImportStore: ObservableObject { } func markFailed(fileId: Int64, error: String) { - catalog.markFailed(id: fileId, message: error) + catalog.markFailedAt(id: fileId, message: error, processedAtMillis: currentTimeMillis()) load() } @@ -577,7 +605,10 @@ final class IosAudioImportStore: ObservableObject { status: IosImportedAudioStatus(sharedState: file.state), transcriptText: file.transcriptText, durationUs: file.durationUs?.int64Value, - lastError: file.lastError + lastError: file.lastError, + processedAt: file.processedAtMillis.map { + Date(timeIntervalSince1970: Double($0.int64Value) / 1000) + } ) } diff --git a/iosApp/VoiceInbox/IosMainScreenShellState.swift b/iosApp/VoiceInbox/IosMainScreenShellState.swift index 5d846da..5fab936 100644 --- a/iosApp/VoiceInbox/IosMainScreenShellState.swift +++ b/iosApp/VoiceInbox/IosMainScreenShellState.swift @@ -4,225 +4,204 @@ import Shared enum IosShellCatalogSelection: String, CaseIterable, Identifiable { case new case processed + case all var id: String { rawValue } var title: String { switch self { - case .new: - "New" - case .processed: - "Processed" + case .new: "New" + case .processed: "Processed" + case .all: "All" } } - var sharedTab: MainScreenCatalogTab { + var sharedFilter: TaskListFilter { switch self { - case .new: - MainScreenCatalogTab.theNew - case .processed: - MainScreenCatalogTab.processed + case .new: .theNew + case .processed: .processed + case .all: .all } } } -struct IosShellMainScreen { - let state: MainScreenState - let rows: [IosShellAudioRow] +struct IosTaskListScreen { + let state: TaskListState + let filesById: [Int64: IosImportedAudioFile] } -struct IosShellAudioRow: Identifiable { - let id: Int64 - let title: String - let subtitle: String - let badge: String - let imported: Bool - let status: IosImportedAudioStatus? - let transcriptText: String? - let lastError: String? - let state: MainScreenRowState +enum IosTaskActionRoute: Equatable { + case modelDownload + case modelImport + case modelCancel + case outputSelection + case folderSelection + case folderRefresh + case transcribe + case retry + case play + case stop + case showText + case audioImport +} + +enum IosTaskActionRouter { + static func route(_ kind: TaskActionKind) -> IosTaskActionRoute? { + switch kind { + case .downloadModel, .retryModelDownload: .modelDownload + case .importModel: .modelImport + case .cancelModelDownload: .modelCancel + case .selectOutput: .outputSelection + case .selectFolder: .folderSelection + case .refreshFolder: .folderRefresh + case .transcribe: .transcribe + case .retryTranscription: .retry + case .play: .play + case .stop: .stop + case .showText: .showText + case .importAudio: .audioImport + default: nil + } + } } final class IosMainScreenShellState { func screen( selection: IosShellCatalogSelection, importedFiles: [IosImportedAudioFile], - runtimeReady: Bool, - modelReady: Bool, + modelStatus: IosSpeechModelStatus, + modelMessage: String?, modelInstalling: Bool, modelDownloadAvailable: Bool, modelDownloadProgress: Int?, - modelMessage: String, - outputReady: Bool, + modelCanCancel: Bool, + outputStatus: IosOutputDocumentStatus, + folderStatus: IosInboxFolderStatus, + folderScanning: Bool, activePreviewEntryId: Int64?, previewState: PreviewPlaybackState, - transcription: IosSingleFileTranscriptionState? = nil - ) -> IosShellMainScreen { - let rowsForSelection = displayRows(for: selection, importedFiles: importedFiles) - let rows = rowsForSelection.map { $0.input } - let pendingCount = importedFiles.count { $0.status == .pending } - let state = MainScreenStateController.shared.state( - input: input( - selectedTab: selection.sharedTab, - pendingCount: pendingCount, - displayedRowCount: Int32(rows.count), - runtimeReady: runtimeReady, - modelReady: modelReady, - modelInstalling: modelInstalling, - modelDownloadAvailable: modelDownloadAvailable, - modelDownloadProgress: modelDownloadProgress, - modelMessage: modelMessage, - outputReady: outputReady, - activePreviewEntryId: activePreviewEntryId, - previewState: previewState, - transcription: transcription, - rows: rows + transcription: IosSingleFileTranscriptionState, + preparationOwnerEntryId: Int64?, + prerequisiteError: String?, + actionsEnabled: Bool + ) -> IosTaskListScreen { + let input = TaskListInput( + filter: selection.sharedFilter, + model: modelSnapshot( + status: modelStatus, + message: modelMessage, + installing: modelInstalling, + downloadAvailable: modelDownloadAvailable, + progress: modelDownloadProgress, + canCancel: modelCanCancel + ), + output: outputSnapshot(outputStatus), + folder: folderSnapshot(folderStatus, scanning: folderScanning), + audio: importedFiles.map { file in + AudioTaskSnapshot( + entryId: file.id, + title: file.displayName, + detail: subtitle(for: file), + state: file.status.sharedState, + importedAtMillis: Int64(file.importedAt.timeIntervalSince1970 * 1000), + terminalAtMillis: file.processedAt.map { + KotlinLong(longLong: Int64($0.timeIntervalSince1970 * 1000)) + }, + lastError: file.lastError, + hasTranscriptText: file.transcriptText?.isEmpty == false, + noSpeech: Self.isNoSpeech(file.lastError), + eligibleForTranscription: actionsEnabled + ) + }, + preview: PreviewTaskSnapshot( + activeEntryId: activePreviewEntryId.map(KotlinLong.init(longLong:)), + state: previewState + ), + transcription: TranscriptionTaskSnapshot( + active: transcription.active, + activeEntryId: transcription.fileId.map(KotlinLong.init(longLong:)), + preparationOwnerEntryId: preparationOwnerEntryId.map(KotlinLong.init(longLong:)), + phase: transcription.phase, + percent: transcription.progressPercent.map { KotlinInt(int: Int32($0)) }, + processedUs: transcription.processedUs > 0 ? KotlinLong(longLong: transcription.processedUs) : nil, + durationUs: transcription.durationUs > 0 ? KotlinLong(longLong: transcription.durationUs) : nil, + completedFiles: transcription.totalFiles > 0 ? KotlinInt(int: transcription.completedFiles) : nil, + totalFiles: transcription.totalFiles > 0 ? KotlinInt(int: transcription.totalFiles) : nil, + failedFiles: transcription.failedFiles > 0 ? KotlinInt(int: transcription.failedFiles) : nil, + prerequisiteError: prerequisiteError ) ) - let sharedRowsById = Dictionary(uniqueKeysWithValues: state.rows.map { ($0.entryId, $0) }) - let displayRows = rowsForSelection.compactMap { row -> IosShellAudioRow? in - guard let rowState = sharedRowsById[row.input.entryId] else { return nil } - return IosShellAudioRow( - id: row.input.entryId, - title: row.title, - subtitle: row.subtitle, - badge: row.badge, - imported: row.imported, - status: row.status, - transcriptText: row.transcriptText, - lastError: row.lastError, - state: rowState - ) + return IosTaskListScreen( + state: TaskListPresentationController.shared.state(input: input), + filesById: Dictionary(uniqueKeysWithValues: importedFiles.map { ($0.id, $0) }) + ) + } + + private func modelSnapshot( + status: IosSpeechModelStatus, + message: String?, + installing: Bool, + downloadAvailable: Bool, + progress: Int?, + canCancel: Bool + ) -> ModelSetupSnapshot { + let state: ModelSetupSnapshotState + if installing { + state = .installing + } else if status.isReady { + state = .ready + } else if status.installationState == .invalid { + state = .invalid + } else { + state = .required } - return IosShellMainScreen( + return ModelSetupSnapshot( state: state, - rows: displayRows + detail: status.detail ?? message, + progressPercent: progress.map { KotlinInt(int: Int32($0)) }, + downloadAvailable: downloadAvailable, + canCancel: canCancel ) } - private func displayRows(for selection: IosShellCatalogSelection, importedFiles: [IosImportedAudioFile]) -> [DisplayRow] { - switch selection { - case .new: - importedFiles.filter { $0.status == .pending || $0.status == .processing }.map { file in - DisplayRow( - input: MainScreenRowInput( - entryId: file.id, - state: file.status.sharedState, - hasTranscriptText: file.transcriptText?.isEmpty == false - ), - title: file.displayName, - subtitle: subtitle(for: file), - badge: file.status.badge, - imported: true, - status: file.status, - transcriptText: file.transcriptText, - lastError: file.lastError - ) - } - case .processed: - importedFiles.filter { $0.status == .processed || $0.status == .failed }.map { file in - DisplayRow( - input: MainScreenRowInput( - entryId: file.id, - state: file.status.sharedState, - hasTranscriptText: file.transcriptText?.isEmpty == false - ), - title: file.displayName, - subtitle: subtitle(for: file), - badge: file.status.badge, - imported: true, - status: file.status, - transcriptText: file.transcriptText, - lastError: file.lastError - ) - } + private func outputSnapshot(_ status: IosOutputDocumentStatus) -> OutputSetupSnapshot { + let state: OutputSetupSnapshotState = status.ready + ? .ready + : (status.displayName == nil ? .required : .invalid) + return OutputSetupSnapshot(state: state, detail: status.message) + } + + private func folderSnapshot( + _ status: IosInboxFolderStatus, + scanning: Bool + ) -> FolderSetupSnapshot { + let state: FolderSetupSnapshotState + if scanning { + state = .scanning + } else if status.hasError { + state = .error + } else if status.needsSelection, status.displayName != nil { + state = .error + } else if status.needsSelection { + state = .unselected + } else { + state = .ready } + return FolderSetupSnapshot(state: state, detail: status.message) } private func subtitle(for file: IosImportedAudioFile) -> String { var parts = ["Imported", file.formattedSize] - if let lastError = file.lastError, !lastError.isEmpty { - parts.append(lastError) - } else if let durationUs = file.durationUs, durationUs > 0 { - parts.append(formatDuration(microseconds: durationUs)) + if let durationUs = file.durationUs, durationUs > 0 { + let totalSeconds = durationUs / 1_000_000 + parts.append("\(totalSeconds / 60):\(String(format: "%02d", totalSeconds % 60))") } return parts.joined(separator: " • ") } - private func formatDuration(microseconds: Int64) -> String { - let totalSeconds = microseconds / 1_000_000 - let minutes = totalSeconds / 60 - let seconds = String(format: "%02d", totalSeconds % 60) - return "\(minutes):\(seconds)" - } - - private func input( - selectedTab: MainScreenCatalogTab, - pendingCount: Int, - displayedRowCount: Int32, - runtimeReady: Bool, - modelReady: Bool, - modelInstalling: Bool, - modelDownloadAvailable: Bool, - modelDownloadProgress: Int?, - modelMessage: String, - outputReady: Bool, - activePreviewEntryId: Int64?, - previewState: PreviewPlaybackState, - transcription: IosSingleFileTranscriptionState?, - rows: [MainScreenRowInput] - ) -> MainScreenInput { - let active = transcription?.active == true - let processedUs = transcription?.processedUs ?? 0 - let durationUs = transcription?.durationUs ?? 0 - let installationState: SpeechModelInstallationState = modelInstalling - ? .installing - : (modelReady && runtimeReady ? .installed : .notInstalled) - let runtimeState: SpeechModelRuntimeState = transcription?.active == true && - transcription?.phase?.localizedCaseInsensitiveContains("model") == true - ? .loading - : .unloaded - return MainScreenInput( - modelMessage: modelMessage, - modelInstallationState: installationState, - modelRuntimeState: runtimeState, - modelDownloadAvailable: modelDownloadAvailable, - modelDownloadProgress: modelDownloadProgress.map { KotlinInt(int: Int32($0)) }, - - outputSelected: outputReady, - folderSelected: true, - pendingCount: Int32(pendingCount), - folderChecking: false, - folderScanQueued: false, - scanning: false, - scanMessage: nil, - transcriptionState: active ? TranscriptionObservationState.active : TranscriptionObservationState.idle, - transcriptionPhase: transcription?.phase, - transcriptionFilename: transcription?.fileName, - transcriptionIndeterminate: active && transcription?.progressPercent == nil, - transcriptionProgress: Int32(transcription?.progressPercent ?? 0), - processedUs: processedUs, - durationUs: durationUs, - completedFiles: transcription?.completedFiles ?? 0, - totalFiles: transcription?.totalFiles ?? 0, - failedFiles: transcription?.failedFiles ?? 0, - errorMessage: nil, - selectedTab: selectedTab, - displayedRowCount: displayedRowCount, - activePreviewEntryId: activePreviewEntryId.map { KotlinLong(longLong: $0) }, - previewState: previewState, - rows: rows - ) - } - - private struct DisplayRow { - let input: MainScreenRowInput - let title: String - let subtitle: String - let badge: String - let imported: Bool - let status: IosImportedAudioStatus? - let transcriptText: String? - let lastError: String? + static func isNoSpeech(_ message: String?) -> Bool { + guard let message else { return false } + return message.localizedCaseInsensitiveContains("no text") || + message.localizedCaseInsensitiveContains("no speech") } } diff --git a/iosApp/VoiceInbox/IosSingleFileTranscriptionController.swift b/iosApp/VoiceInbox/IosSingleFileTranscriptionController.swift index 41a27f4..9345541 100644 --- a/iosApp/VoiceInbox/IosSingleFileTranscriptionController.swift +++ b/iosApp/VoiceInbox/IosSingleFileTranscriptionController.swift @@ -74,6 +74,8 @@ struct IosSingleFileTranscriptionState { @MainActor final class IosSingleFileTranscriptionController: ObservableObject { @Published private(set) var state = IosSingleFileTranscriptionState.idle + @Published private(set) var preparationOwnerFileId: Int64? + @Published private(set) var prerequisiteError: String? @Published var message: String? private var task: Task? @@ -101,6 +103,8 @@ final class IosSingleFileTranscriptionController: ObservableObject { onSuccess: ((String) -> Void)? = nil ) { task?.cancel() + preparationOwnerFileId = file.id + prerequisiteError = nil state = IosSingleFileTranscriptionState( active: true, fileId: file.id, @@ -122,7 +126,9 @@ final class IosSingleFileTranscriptionController: ObservableObject { claim: { store.markProcessing(fileId: file.id) } ) guard prepared else { - message = modelStore.message ?? "Speech model preparation failed." + let error = modelStore.message ?? "Speech model preparation failed." + prerequisiteError = error + message = error state = .idle return } @@ -139,6 +145,7 @@ final class IosSingleFileTranscriptionController: ObservableObject { modelPrepared: true ) { progress in Task { @MainActor in + self.preparationOwnerFileId = nil self.state = IosSingleFileTranscriptionState( active: true, fileId: file.id, @@ -175,6 +182,8 @@ final class IosSingleFileTranscriptionController: ObservableObject { store.markFailed(fileId: file.id, error: error) message = error } + preparationOwnerFileId = nil + prerequisiteError = nil state = .idle } } @@ -188,6 +197,14 @@ final class IosSingleFileTranscriptionController: ObservableObject { ) { guard !state.active else { return } task?.cancel() + preparationOwnerFileId = store.files + .filter { $0.status == .pending } + .sorted { + if $0.importedAt != $1.importedAt { return $0.importedAt < $1.importedAt } + return $0.id < $1.id + } + .first?.id + prerequisiteError = nil state = IosSingleFileTranscriptionState( active: true, fileId: nil, @@ -205,7 +222,9 @@ final class IosSingleFileTranscriptionController: ObservableObject { task = Task { guard await modelStore.prepareForTranscription() != nil else { - message = modelStore.message ?? "Speech model preparation failed." + let error = modelStore.message ?? "Speech model preparation failed." + prerequisiteError = error + message = error state = .idle return } @@ -219,9 +238,10 @@ final class IosSingleFileTranscriptionController: ObservableObject { outputDocument: outputDocument ) { progress in Task { @MainActor in + self.preparationOwnerFileId = nil self.state = IosSingleFileTranscriptionState( active: true, - fileId: nil, + fileId: progress.activeEntryId?.int64Value, fileName: progress.filename, phase: progress.phase, processedUs: progress.processedUs?.int64Value ?? 0, @@ -243,6 +263,8 @@ final class IosSingleFileTranscriptionController: ObservableObject { } message = outcome.summary + preparationOwnerFileId = nil + prerequisiteError = nil state = IosSingleFileTranscriptionState( active: false, fileId: nil, diff --git a/iosApp/VoiceInbox/IosSpeechModelStore.swift b/iosApp/VoiceInbox/IosSpeechModelStore.swift index e70e70d..29d68f4 100644 --- a/iosApp/VoiceInbox/IosSpeechModelStore.swift +++ b/iosApp/VoiceInbox/IosSpeechModelStore.swift @@ -145,6 +145,10 @@ final class IosSpeechModelStore: ObservableObject { isInstalling || downloadTask != nil } + var canCancelDownload: Bool { + downloadTask != nil + } + func reload() { status = inspectInstallation(installationDirectory) } diff --git a/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift b/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift index 5b710de..610746d 100644 --- a/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift +++ b/iosApp/VoiceInboxTests/DeferredSpeechModelLoadingTests.swift @@ -128,6 +128,119 @@ final class DeferredSpeechModelLoadingTests: XCTestCase { XCTAssertEqual(prepares.value, 1) XCTAssertEqual(claims.value, 0) } + + @MainActor + func testTaskListAdapterSupportsAllAndKeepsOptionalUnselectedFolderOutOfTasks() { + let modelStatus = IosSpeechModelStatus( + directory: FileManager.default.temporaryDirectory, + installationState: .installedVerified, + missingFiles: [] + ) + let files = [ + IosImportedAudioFile( + id: 1, + displayName: "new.m4a", + localFileName: "new.m4a", + sizeBytes: 10, + importedAt: Date(timeIntervalSince1970: 100), + status: .pending + ), + IosImportedAudioFile( + id: 2, + displayName: "failed.m4a", + localFileName: "failed.m4a", + sizeBytes: 20, + importedAt: Date(timeIntervalSince1970: 200), + status: .failed, + lastError: "decode failed", + processedAt: Date(timeIntervalSince1970: 300) + ), + ] + + let screen = IosMainScreenShellState().screen( + selection: .all, + importedFiles: files, + modelStatus: modelStatus, + modelMessage: nil, + modelInstalling: false, + modelDownloadAvailable: false, + modelDownloadProgress: nil, + modelCanCancel: false, + outputStatus: IosOutputDocumentStatus(displayName: "notes.md", message: "Ready", ready: true), + folderStatus: IosInboxFolderStatus(displayName: nil, message: nil, needsSelection: true), + folderScanning: false, + activePreviewEntryId: nil, + previewState: .idle, + transcription: .idle, + preparationOwnerEntryId: nil, + prerequisiteError: nil, + actionsEnabled: true + ) + + XCTAssertEqual(screen.state.tasks.map(\.stableId), ["audio:2", "audio:1"]) + XCTAssertFalse(screen.state.tasks.contains { $0.stableId == "setup:model" }) + XCTAssertFalse(screen.state.tasks.contains { $0.stableId == "setup:folder" }) + XCTAssertEqual(screen.filesById.count, 2) + } + + @MainActor + func testTaskListAdapterAttachesPreparationFailureToPendingOwner() { + let file = IosImportedAudioFile( + id: 9, + displayName: "voice.m4a", + localFileName: "voice.m4a", + sizeBytes: 10, + importedAt: Date(), + status: .pending + ) + let screen = IosMainScreenShellState().screen( + selection: .new, + importedFiles: [file], + modelStatus: IosSpeechModelStatus( + directory: FileManager.default.temporaryDirectory, + installationState: .installedVerified, + missingFiles: [] + ), + modelMessage: nil, + modelInstalling: false, + modelDownloadAvailable: false, + modelDownloadProgress: nil, + modelCanCancel: false, + outputStatus: IosOutputDocumentStatus(displayName: "notes.md", message: "Ready", ready: true), + folderStatus: IosInboxFolderStatus(displayName: nil, message: nil, needsSelection: true), + folderScanning: false, + activePreviewEntryId: nil, + previewState: .idle, + transcription: .idle, + preparationOwnerEntryId: 9, + prerequisiteError: "Model could not be loaded", + actionsEnabled: true + ) + + guard let task = screen.state.tasks.first as? AudioTaskPresentation else { + return XCTFail("Expected an audio task") + } + XCTAssertEqual(task.state, .pending) + XCTAssertEqual(task.errorMessage, "Model could not be loaded") + } + + func testTypedActionsHaveExplicitIosRoutes() { + XCTAssertEqual(IosTaskActionRouter.route(.downloadModel), .modelDownload) + XCTAssertEqual(IosTaskActionRouter.route(.selectOutput), .outputSelection) + XCTAssertEqual(IosTaskActionRouter.route(.selectFolder), .folderSelection) + XCTAssertEqual(IosTaskActionRouter.route(.transcribe), .transcribe) + XCTAssertEqual(IosTaskActionRouter.route(.retryTranscription), .retry) + XCTAssertEqual(IosTaskActionRouter.route(.showText), .showText) + } + + func testRoutineImportAndScanSummariesDoNotRequestAnAlert() { + XCTAssertNil(IosAudioImportSummary(imported: 2, skipped: 0, failed: 0).alertMessage) + XCTAssertNil(IosAudioImportSummary(imported: 0, skipped: 4, failed: 0).alertMessage) + XCTAssertEqual( + IosAudioImportSummary(imported: 1, skipped: 0, failed: 1).alertMessage, + "1 imported, 1 failed" + ) + } } private final class LockedCounter: @unchecked Sendable { diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/AudioCatalog.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/AudioCatalog.kt index da89fbc..372bbda 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/AudioCatalog.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/AudioCatalog.kt @@ -48,6 +48,10 @@ interface AudioCatalogQueuePort { fun markFailed(id: Long, message: String) + fun markFailedAt(id: Long, message: String, processedAtMillis: Long) { + markFailed(id, message) + } + fun markPending(id: Long) } diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCase.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCase.kt index e58d1d3..199e576 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCase.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCase.kt @@ -11,6 +11,7 @@ data class BatchTranscriptionInput( data class BatchTranscriptionProgress( val phase: String, + val activeEntryId: Long? = null, val filename: String? = null, val completed: Int, val total: Int, @@ -83,6 +84,7 @@ class BatchTranscriptionUseCase( onProgress( BatchTranscriptionProgress( phase = progress.phase, + activeEntryId = entry.id, filename = entry.displayName, completed = completed, total = total, @@ -106,7 +108,11 @@ class BatchTranscriptionUseCase( currentEntry = null throw cancelled } catch (error: Throwable) { - catalog.markFailed(entry.id, error.message ?: ERROR_TRANSCRIPTION_FAILED) + catalog.markFailedAt( + entry.id, + error.message ?: ERROR_TRANSCRIPTION_FAILED, + clock.currentTimeMillis(), + ) failed += 1 } completed += 1 @@ -150,6 +156,7 @@ class BatchTranscriptionUseCase( failed: Int, ): BatchTranscriptionProgress = BatchTranscriptionProgress( phase = BatchTranscriptionRules.summary(completed, total, failed), + activeEntryId = null, filename = null, completed = completed, total = total, diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SqlDelightAudioCatalogRepository.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SqlDelightAudioCatalogRepository.kt index 0aa33ca..1bbe861 100644 --- a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SqlDelightAudioCatalogRepository.kt +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/SqlDelightAudioCatalogRepository.kt @@ -183,7 +183,11 @@ class SqlDelightAudioCatalogRepository( } override fun markFailed(id: Long, message: String) { - queries.markFailed(last_error = message, id = id) + queries.markFailed(last_error = message, processed_at = null, id = id) + } + + override fun markFailedAt(id: Long, message: String, processedAtMillis: Long) { + queries.markFailed(last_error = message, processed_at = processedAtMillis, id = id) } override fun markPending(id: Long) { diff --git a/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationController.kt b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationController.kt new file mode 100644 index 0000000..a37806b --- /dev/null +++ b/shared/src/commonMain/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationController.kt @@ -0,0 +1,431 @@ +package me.maxistar.voiceinbox.core + +enum class TaskListFilter { + NEW, + PROCESSED, + ALL, +} + +enum class SetupTaskKind { + MODEL, + OUTPUT, + FOLDER, +} + +enum class SetupTaskState { + REQUIRED, + ACTIVE, + ERROR, +} + +enum class AudioTaskState { + PENDING, + PROCESSING, + SUCCEEDED, + FAILED, + NO_SPEECH, +} + +enum class TaskRetention { + UNTIL_COMPLETED, + RETAINED, +} + +enum class TaskActionKind { + DOWNLOAD_MODEL, + IMPORT_MODEL, + CANCEL_MODEL_DOWNLOAD, + RETRY_MODEL_DOWNLOAD, + SELECT_OUTPUT, + SELECT_FOLDER, + REFRESH_FOLDER, + TRANSCRIBE, + RETRY_TRANSCRIPTION, + PLAY, + STOP, + SHOW_TEXT, + IMPORT_AUDIO, +} + +data class TaskActionPresentation( + val kind: TaskActionKind, + val label: String, + val enabled: Boolean = true, +) + +data class TaskProgressPresentation( + val phase: String, + val percent: Int? = null, + val processedUs: Long? = null, + val durationUs: Long? = null, + val completedFiles: Int? = null, + val totalFiles: Int? = null, + val failedFiles: Int? = null, +) + +sealed class TaskPresentation { + abstract val stableId: String + abstract val title: String + abstract val detail: String? + abstract val badge: String + abstract val progress: TaskProgressPresentation? + abstract val errorMessage: String? + abstract val actions: List + abstract val retention: TaskRetention +} + +data class SetupTaskPresentation( + override val stableId: String, + val kind: SetupTaskKind, + val state: SetupTaskState, + override val title: String, + override val detail: String?, + override val badge: String, + override val progress: TaskProgressPresentation?, + override val errorMessage: String?, + override val actions: List, + override val retention: TaskRetention = TaskRetention.UNTIL_COMPLETED, +) : TaskPresentation() + +data class AudioTaskPresentation( + override val stableId: String, + val entryId: Long, + val state: AudioTaskState, + override val title: String, + override val detail: String?, + override val badge: String, + override val progress: TaskProgressPresentation?, + override val errorMessage: String?, + override val actions: List, + override val retention: TaskRetention = TaskRetention.RETAINED, +) : TaskPresentation() + +enum class ModelSetupSnapshotState { + REQUIRED, + INSTALLING, + INVALID, + READY, +} + +data class ModelSetupSnapshot( + val state: ModelSetupSnapshotState, + val detail: String? = null, + val progressPercent: Int? = null, + val downloadAvailable: Boolean = false, + val canCancel: Boolean = false, +) + +enum class OutputSetupSnapshotState { + REQUIRED, + INVALID, + READY, +} + +data class OutputSetupSnapshot( + val state: OutputSetupSnapshotState, + val detail: String? = null, +) + +enum class FolderSetupSnapshotState { + UNSELECTED, + READY, + SCANNING, + ERROR, +} + +data class FolderSetupSnapshot( + val state: FolderSetupSnapshotState, + val detail: String? = null, +) + +data class AudioTaskSnapshot( + val entryId: Long, + val title: String, + val detail: String? = null, + val state: AudioFileState, + val importedAtMillis: Long, + val terminalAtMillis: Long? = null, + val lastError: String? = null, + val hasTranscriptText: Boolean = false, + val noSpeech: Boolean = false, + val eligibleForTranscription: Boolean = true, +) + +data class PreviewTaskSnapshot( + val activeEntryId: Long? = null, + val state: PreviewPlaybackState = PreviewPlaybackState.IDLE, +) + +data class TranscriptionTaskSnapshot( + val active: Boolean = false, + val activeEntryId: Long? = null, + val preparationOwnerEntryId: Long? = null, + val phase: String? = null, + val percent: Int? = null, + val processedUs: Long? = null, + val durationUs: Long? = null, + val completedFiles: Int? = null, + val totalFiles: Int? = null, + val failedFiles: Int? = null, + val prerequisiteError: String? = null, +) + +data class TaskListInput( + val filter: TaskListFilter, + val model: ModelSetupSnapshot, + val output: OutputSetupSnapshot, + val folder: FolderSetupSnapshot, + val audio: List, + val preview: PreviewTaskSnapshot = PreviewTaskSnapshot(), + val transcription: TranscriptionTaskSnapshot = TranscriptionTaskSnapshot(), +) + +data class TaskListBatchActionState( + val visible: Boolean, + val enabled: Boolean, + val eligibleCount: Int, +) + +data class TaskListState( + val filter: TaskListFilter, + val tasks: List, + val emptyMessage: String?, + val emptyActions: List, + val batchAction: TaskListBatchActionState, +) + +object TaskListPresentationController { + fun state(input: TaskListInput): TaskListState { + val setupTasks = setupTasks(input) + val audioTasks = input.audio + .filter { visibleInFilter(it.state, input.filter) } + .sortedWith(audioComparator(input.filter)) + .map { audioTask(it, input) } + val visibleSetup = if (input.filter == TaskListFilter.PROCESSED) emptyList() else setupTasks + val tasks = visibleSetup + audioTasks + val eligibleCount = input.audio.count { + it.state == AudioFileState.PENDING && it.eligibleForTranscription + } + val batchVisible = input.filter == TaskListFilter.NEW && eligibleCount > 0 + return TaskListState( + filter = input.filter, + tasks = tasks, + emptyMessage = if (tasks.isEmpty()) emptyMessage(input.filter) else null, + emptyActions = if (tasks.isEmpty() && input.filter != TaskListFilter.PROCESSED) { + listOf( + TaskActionPresentation(TaskActionKind.IMPORT_AUDIO, "Import Audio Files"), + TaskActionPresentation(TaskActionKind.SELECT_FOLDER, "Select Audio Folder"), + ) + } else { + emptyList() + }, + batchAction = TaskListBatchActionState( + visible = batchVisible, + enabled = batchVisible && !input.transcription.active, + eligibleCount = eligibleCount, + ), + ) + } + + fun preparationOwnerEntryId(audio: List): Long? = audio + .asSequence() + .filter { it.state == AudioFileState.PENDING && it.eligibleForTranscription } + .sortedWith(compareBy({ it.importedAtMillis }, { it.title.lowercase() }, { it.entryId })) + .firstOrNull() + ?.entryId + + private fun setupTasks(input: TaskListInput): List = buildList { + modelTask(input.model)?.let(::add) + outputTask(input.output)?.let(::add) + folderTask(input.folder)?.let(::add) + } + + private fun modelTask(snapshot: ModelSetupSnapshot): SetupTaskPresentation? { + if (snapshot.state == ModelSetupSnapshotState.READY) return null + val active = snapshot.state == ModelSetupSnapshotState.INSTALLING + val error = snapshot.state == ModelSetupSnapshotState.INVALID + val actions = when { + active && snapshot.canCancel -> listOf( + TaskActionPresentation(TaskActionKind.CANCEL_MODEL_DOWNLOAD, "Cancel"), + ) + active -> emptyList() + error -> listOf( + TaskActionPresentation(TaskActionKind.RETRY_MODEL_DOWNLOAD, "Retry Download", snapshot.downloadAvailable), + TaskActionPresentation(TaskActionKind.IMPORT_MODEL, "Install Manually"), + ) + else -> buildList { + if (snapshot.downloadAvailable) { + add(TaskActionPresentation(TaskActionKind.DOWNLOAD_MODEL, "Download Model")) + } + add(TaskActionPresentation(TaskActionKind.IMPORT_MODEL, "Install Manually")) + } + } + return SetupTaskPresentation( + stableId = "setup:model", + kind = SetupTaskKind.MODEL, + state = when { + active -> SetupTaskState.ACTIVE + error -> SetupTaskState.ERROR + else -> SetupTaskState.REQUIRED + }, + title = "Install Speech Model", + detail = snapshot.detail, + badge = if (active) "Downloading" else if (error) "Needs attention" else "Required", + progress = if (active) TaskProgressPresentation("Downloading model", snapshot.progressPercent) else null, + errorMessage = snapshot.detail.takeIf { error }, + actions = actions, + ) + } + + private fun outputTask(snapshot: OutputSetupSnapshot): SetupTaskPresentation? { + if (snapshot.state == OutputSetupSnapshotState.READY) return null + val error = snapshot.state == OutputSetupSnapshotState.INVALID + return SetupTaskPresentation( + stableId = "setup:output", + kind = SetupTaskKind.OUTPUT, + state = if (error) SetupTaskState.ERROR else SetupTaskState.REQUIRED, + title = "Select Output File", + detail = snapshot.detail, + badge = if (error) "Needs attention" else "Required", + progress = null, + errorMessage = snapshot.detail.takeIf { error }, + actions = listOf(TaskActionPresentation(TaskActionKind.SELECT_OUTPUT, "Select Output File")), + ) + } + + private fun folderTask(snapshot: FolderSetupSnapshot): SetupTaskPresentation? = when (snapshot.state) { + FolderSetupSnapshotState.UNSELECTED, + FolderSetupSnapshotState.READY, + -> null + FolderSetupSnapshotState.SCANNING -> SetupTaskPresentation( + stableId = "setup:folder", + kind = SetupTaskKind.FOLDER, + state = SetupTaskState.ACTIVE, + title = "Refresh Audio Folder", + detail = snapshot.detail, + badge = "Scanning", + progress = TaskProgressPresentation("Scanning audio folder"), + errorMessage = null, + actions = emptyList(), + ) + FolderSetupSnapshotState.ERROR -> SetupTaskPresentation( + stableId = "setup:folder", + kind = SetupTaskKind.FOLDER, + state = SetupTaskState.ERROR, + title = "Restore Audio Folder Access", + detail = snapshot.detail, + badge = "Needs attention", + progress = null, + errorMessage = snapshot.detail, + actions = listOf( + TaskActionPresentation(TaskActionKind.SELECT_FOLDER, "Select Folder"), + TaskActionPresentation(TaskActionKind.REFRESH_FOLDER, "Retry"), + ), + ) + } + + private fun audioTask(snapshot: AudioTaskSnapshot, input: TaskListInput): AudioTaskPresentation { + val progressOwnerEntryId = input.transcription.activeEntryId + ?: input.transcription.preparationOwnerEntryId + ?: preparationOwnerEntryId(input.audio).takeIf { + input.transcription.active && input.transcription.phase?.contains("model", ignoreCase = true) == true + } + val ownsProgress = input.transcription.active && ( + progressOwnerEntryId == snapshot.entryId + ) + val presentationState = when (snapshot.state) { + AudioFileState.PENDING -> if (ownsProgress) AudioTaskState.PROCESSING else AudioTaskState.PENDING + AudioFileState.PROCESSING -> AudioTaskState.PROCESSING + AudioFileState.PROCESSED -> AudioTaskState.SUCCEEDED + AudioFileState.FAILED -> if (snapshot.noSpeech) AudioTaskState.NO_SPEECH else AudioTaskState.FAILED + AudioFileState.MISSING -> AudioTaskState.FAILED + } + val isPreviewing = input.preview.activeEntryId == snapshot.entryId && + input.preview.state != PreviewPlaybackState.IDLE + val actions = buildList { + when (presentationState) { + AudioTaskState.PENDING -> add( + TaskActionPresentation( + TaskActionKind.TRANSCRIBE, + "Transcribe", + snapshot.eligibleForTranscription && !input.transcription.active, + ), + ) + AudioTaskState.FAILED, + AudioTaskState.NO_SPEECH, + -> add( + TaskActionPresentation( + TaskActionKind.RETRY_TRANSCRIPTION, + "Retry", + snapshot.eligibleForTranscription && !input.transcription.active, + ), + ) + AudioTaskState.SUCCEEDED -> if (snapshot.hasTranscriptText) { + add(TaskActionPresentation(TaskActionKind.SHOW_TEXT, "Show Text")) + } + AudioTaskState.PROCESSING -> Unit + } + add( + TaskActionPresentation( + if (isPreviewing) TaskActionKind.STOP else TaskActionKind.PLAY, + if (isPreviewing) "Stop" else "Play", + !input.transcription.active, + ), + ) + } + val prerequisiteError = input.transcription.prerequisiteError + .takeIf { + snapshot.state == AudioFileState.PENDING && + (input.transcription.preparationOwnerEntryId ?: preparationOwnerEntryId(input.audio)) == snapshot.entryId + } + return AudioTaskPresentation( + stableId = "audio:${snapshot.entryId}", + entryId = snapshot.entryId, + state = presentationState, + title = snapshot.title, + detail = snapshot.detail, + badge = when (presentationState) { + AudioTaskState.PENDING -> "New" + AudioTaskState.PROCESSING -> "Processing" + AudioTaskState.SUCCEEDED -> "Processed" + AudioTaskState.FAILED -> "Failed" + AudioTaskState.NO_SPEECH -> "No speech" + }, + progress = if (ownsProgress) input.transcription.toProgress() else null, + errorMessage = prerequisiteError ?: snapshot.lastError, + actions = actions, + ) + } + + private fun TranscriptionTaskSnapshot.toProgress(): TaskProgressPresentation = + TaskProgressPresentation( + phase = phase ?: "Processing", + percent = percent, + processedUs = processedUs, + durationUs = durationUs, + completedFiles = completedFiles, + totalFiles = totalFiles, + failedFiles = failedFiles, + ) + + private fun visibleInFilter(state: AudioFileState, filter: TaskListFilter): Boolean = when (filter) { + TaskListFilter.NEW -> state == AudioFileState.PENDING || state == AudioFileState.PROCESSING + TaskListFilter.PROCESSED -> state == AudioFileState.PROCESSED || state == AudioFileState.FAILED + TaskListFilter.ALL -> true + } + + private fun audioComparator(filter: TaskListFilter): Comparator = when (filter) { + TaskListFilter.PROCESSED -> compareByDescending { + it.terminalAtMillis ?: it.importedAtMillis + }.thenByDescending { it.importedAtMillis }.thenByDescending { it.entryId } + TaskListFilter.NEW, + TaskListFilter.ALL, + -> compareByDescending { it.importedAtMillis }.thenByDescending { it.entryId } + } + + private fun emptyMessage(filter: TaskListFilter): String = when (filter) { + TaskListFilter.NEW -> "No new tasks" + TaskListFilter.PROCESSED -> "No processed audio files" + TaskListFilter.ALL -> "No audio tasks" + } +} diff --git a/shared/src/commonMain/sqldelight/me/maxistar/voiceinbox/db/AudioFiles.sq b/shared/src/commonMain/sqldelight/me/maxistar/voiceinbox/db/AudioFiles.sq index 99d2800..5902267 100644 --- a/shared/src/commonMain/sqldelight/me/maxistar/voiceinbox/db/AudioFiles.sq +++ b/shared/src/commonMain/sqldelight/me/maxistar/voiceinbox/db/AudioFiles.sq @@ -215,7 +215,7 @@ markFailed: UPDATE audio_files SET state = 'FAILED', last_error = ?, - processed_at = NULL, + processed_at = ?, transcript_text = NULL, duration_us = NULL WHERE id = ?; diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCaseTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCaseTest.kt index 02e0b7e..73d06ce 100644 --- a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCaseTest.kt +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/BatchTranscriptionUseCaseTest.kt @@ -69,6 +69,7 @@ class BatchTranscriptionUseCaseTest { assertEquals(AudioFileState.FAILED, catalog.entry(1).state) assertEquals("decode failed", catalog.entry(1).lastError) + assertEquals(1234, catalog.entry(1).processedAtMillis) assertEquals(AudioFileState.PROCESSED, catalog.entry(2).state) assertEquals(BatchTranscriptionResult(2, 2, 1, "Completed 2 of 2, 1 failed", 100, false), result) assertEquals("Completed 1 of 2, 1 failed", progress.first { it.filename == null }.phase) @@ -103,6 +104,7 @@ class BatchTranscriptionUseCaseTest { assertEquals( BatchTranscriptionProgress( phase = "Transcribing", + activeEntryId = 1, filename = "one.wav", completed = 0, total = 1, @@ -113,6 +115,8 @@ class BatchTranscriptionUseCaseTest { ), progress.first(), ) + assertEquals(1L, progress.first().activeEntryId) + assertNull(progress.last().activeEntryId) } private fun useCase( @@ -182,6 +186,11 @@ class BatchTranscriptionUseCaseTest { } } + override fun markFailedAt(id: Long, message: String, processedAtMillis: Long) { + markFailed(id, message) + update(id) { it.copy(processedAtMillis = processedAtMillis) } + } + override fun markPending(id: Long) { update(id) { if (it.state == AudioFileState.PROCESSING) { diff --git a/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationControllerTest.kt b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationControllerTest.kt new file mode 100644 index 0000000..a8f710d --- /dev/null +++ b/shared/src/commonTest/kotlin/me/maxistar/voiceinbox/core/TaskListPresentationControllerTest.kt @@ -0,0 +1,199 @@ +package me.maxistar.voiceinbox.core + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertIs +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class TaskListPresentationControllerTest { + @Test + fun setupTasksAreSynthesizedInKindOrderAndCompletedTasksDisappear() { + val state = state( + filter = TaskListFilter.NEW, + model = ModelSetupSnapshot(ModelSetupSnapshotState.REQUIRED, downloadAvailable = true), + output = OutputSetupSnapshot(OutputSetupSnapshotState.INVALID, "Access expired"), + folder = FolderSetupSnapshot(FolderSetupSnapshotState.SCANNING), + ) + + assertEquals(listOf("setup:model", "setup:output", "setup:folder"), state.tasks.map { it.stableId }) + assertEquals(listOf(SetupTaskKind.MODEL, SetupTaskKind.OUTPUT, SetupTaskKind.FOLDER), state.tasks.map { + assertIs(it).kind + }) + + val completed = state( + filter = TaskListFilter.ALL, + model = ModelSetupSnapshot(ModelSetupSnapshotState.READY), + output = OutputSetupSnapshot(OutputSetupSnapshotState.READY), + folder = FolderSetupSnapshot(FolderSetupSnapshotState.READY), + ) + assertTrue(completed.tasks.isEmpty()) + } + + @Test + fun unselectedOptionalFolderDoesNotCreateBlockingTask() { + val state = state(folder = FolderSetupSnapshot(FolderSetupSnapshotState.UNSELECTED)) + + assertTrue(state.tasks.none { it.stableId == "setup:folder" }) + } + + @Test + fun filtersApplyOpenTerminalAndAllRetentionRules() { + val audio = listOf( + audio(1, AudioFileState.PENDING), + audio(2, AudioFileState.PROCESSING), + audio(3, AudioFileState.PROCESSED), + audio(4, AudioFileState.FAILED), + audio(5, AudioFileState.MISSING), + ) + + assertEquals(listOf("audio:2", "audio:1"), state(TaskListFilter.NEW, audio = audio).tasks.map { it.stableId }) + assertEquals(listOf("audio:4", "audio:3"), state(TaskListFilter.PROCESSED, audio = audio).tasks.map { it.stableId }) + assertEquals(listOf("audio:5", "audio:4", "audio:3", "audio:2", "audio:1"), state(TaskListFilter.ALL, audio = audio).tasks.map { it.stableId }) + } + + @Test + fun noSpeechIsTerminalAndRetainsRetry() { + val task = state( + TaskListFilter.PROCESSED, + audio = listOf(audio(1, AudioFileState.FAILED, noSpeech = true, error = "No text was recognized")), + ).tasks.single() + + val audioTask = assertIs(task) + assertEquals(AudioTaskState.NO_SPEECH, audioTask.state) + assertEquals(TaskActionKind.RETRY_TRANSCRIPTION, audioTask.actions.first().kind) + } + + @Test + fun prerequisiteFailureLeavesPendingTaskInNew() { + val state = state( + filter = TaskListFilter.NEW, + audio = listOf(audio(7, AudioFileState.PENDING)), + transcription = TranscriptionTaskSnapshot( + active = false, + preparationOwnerEntryId = 7, + prerequisiteError = "Model could not be loaded", + ), + ) + + val task = assertIs(state.tasks.single()) + assertEquals(AudioTaskState.PENDING, task.state) + assertEquals("Model could not be loaded", task.errorMessage) + assertTrue(state(TaskListFilter.PROCESSED, audio = listOf(audio(7, AudioFileState.PENDING))).tasks.isEmpty()) + } + + @Test + fun progressBelongsOnlyToStableActiveAudioTask() { + val state = state( + audio = listOf(audio(1, AudioFileState.PENDING), audio(2, AudioFileState.PENDING)), + transcription = TranscriptionTaskSnapshot( + active = true, + preparationOwnerEntryId = 1, + phase = "Preparing speech model", + ), + ) + + val byId = state.tasks.associateBy(TaskPresentation::stableId) + val active = assertIs(byId.getValue("audio:1")) + assertEquals(AudioTaskState.PROCESSING, active.state) + assertEquals("Preparing speech model", active.progress?.phase) + assertNull(assertIs(byId.getValue("audio:2")).progress) + assertEquals("audio:1", active.stableId) + } + + @Test + fun preClaimPreparationUsesDeterministicNextEligiblePendingTask() { + val audio = listOf( + audio(3, AudioFileState.PENDING, importedAt = 300), + audio(1, AudioFileState.PENDING, importedAt = 100), + audio(2, AudioFileState.PENDING, importedAt = 50, eligible = false), + ) + + assertEquals(1L, TaskListPresentationController.preparationOwnerEntryId(audio)) + val state = state( + audio = audio, + transcription = TranscriptionTaskSnapshot(active = true, phase = "Preparing speech model"), + ) + assertEquals( + "audio:1", + state.tasks.single { it.progress != null }.stableId, + ) + assertTrue(audio.all { it.state == AudioFileState.PENDING }) + } + + @Test + fun batchActionCountsOnlyEligiblePendingAudioAndOnlyAppearsInNew() { + val audio = listOf( + audio(1, AudioFileState.PENDING, eligible = true), + audio(2, AudioFileState.PENDING, eligible = false), + audio(3, AudioFileState.FAILED, eligible = true), + ) + + val newState = state(TaskListFilter.NEW, audio = audio) + assertTrue(newState.batchAction.visible) + assertTrue(newState.batchAction.enabled) + assertEquals(1, newState.batchAction.eligibleCount) + assertFalse(state(TaskListFilter.ALL, audio = audio).batchAction.visible) + assertFalse(state(TaskListFilter.PROCESSED, audio = audio).batchAction.visible) + } + + @Test + fun orderingUsesImportTimeExceptProcessedUsesTerminalTime() { + val audio = listOf( + audio(1, AudioFileState.PROCESSED, importedAt = 300, terminalAt = 100), + audio(2, AudioFileState.FAILED, importedAt = 100, terminalAt = 400), + ) + + assertEquals(listOf("audio:2", "audio:1"), state(TaskListFilter.PROCESSED, audio = audio).tasks.map { it.stableId }) + assertEquals(listOf("audio:1", "audio:2"), state(TaskListFilter.ALL, audio = audio).tasks.map { it.stableId }) + } + + @Test + fun emptyStateIsSpecificToFilterAndOffersImportOnlyForOpenLists() { + val newState = state(TaskListFilter.NEW) + val processed = state(TaskListFilter.PROCESSED) + + assertEquals("No new tasks", newState.emptyMessage) + assertEquals(listOf(TaskActionKind.IMPORT_AUDIO, TaskActionKind.SELECT_FOLDER), newState.emptyActions.map { it.kind }) + assertEquals("No processed audio files", processed.emptyMessage) + assertTrue(processed.emptyActions.isEmpty()) + } + + private fun state( + filter: TaskListFilter = TaskListFilter.NEW, + model: ModelSetupSnapshot = ModelSetupSnapshot(ModelSetupSnapshotState.READY), + output: OutputSetupSnapshot = OutputSetupSnapshot(OutputSetupSnapshotState.READY), + folder: FolderSetupSnapshot = FolderSetupSnapshot(FolderSetupSnapshotState.READY), + audio: List = emptyList(), + transcription: TranscriptionTaskSnapshot = TranscriptionTaskSnapshot(), + ): TaskListState = TaskListPresentationController.state( + TaskListInput( + filter = filter, + model = model, + output = output, + folder = folder, + audio = audio, + transcription = transcription, + ), + ) + + private fun audio( + id: Long, + state: AudioFileState, + importedAt: Long = id, + terminalAt: Long? = if (state == AudioFileState.PROCESSED || state == AudioFileState.FAILED) id else null, + noSpeech: Boolean = false, + error: String? = null, + eligible: Boolean = true, + ) = AudioTaskSnapshot( + entryId = id, + title = "$id.wav", + state = state, + importedAtMillis = importedAt, + terminalAtMillis = terminalAt, + lastError = error, + noSpeech = noSpeech, + eligibleForTranscription = eligible, + ) +} From fa2625c0be54f5d3b634fa48dd0f151bd316ec2b Mon Sep 17 00:00:00 2001 From: Max Starikov Date: Thu, 16 Jul 2026 09:56:48 +0200 Subject: [PATCH 06/10] add importing feature --- .../AudioShareIntentInstrumentedTest.kt | 158 +++++++ .../MainActivityInstrumentedTest.kt | 308 ++++++++++++ .../TestAudioDocumentsProvider.java | 42 ++ .../TranscriptionWorkerInstrumentedTest.kt | 51 ++ app/src/main/AndroidManifest.xml | 21 +- .../voiceinbox/AudioImportCoordinator.kt | 367 +++++++++++++++ .../voiceinbox/CatalogRefreshPolicy.kt | 35 ++ .../me/maxistar/voiceinbox/MainActivity.kt | 440 ++++++++++++++---- .../voiceinbox/PersistedSelectionAccess.kt | 68 +++ .../ScheduledTranscriptionWorker.kt | 6 +- .../voiceinbox/TranscriptionWorker.kt | 24 +- app/src/main/res/layout/activity_main.xml | 7 + app/src/main/res/values/strings.xml | 7 + app/src/main/res/xml/imported_audio_paths.xml | 6 + .../voiceinbox/AudioImportCoordinatorTest.kt | 190 ++++++++ .../voiceinbox/CatalogRefreshPolicyTest.kt | 72 +++ .../PersistedSelectionAccessPolicyTest.kt | 55 +++ ...IosSingleFileTranscriptionController.swift | 4 +- .../maxistar/voiceinbox/core/AudioCatalog.kt | 38 +- .../core/BatchTranscriptionUseCase.kt | 8 +- .../core/MainScreenStateController.kt | 15 +- .../core/SqlDelightAudioCatalogRepository.kt | 40 +- .../voiceinbox/core/TranscriptionUiRules.kt | 7 +- .../me/maxistar/voiceinbox/db/AudioFiles.sq | 54 +++ .../core/AudioCatalogQueuePortTest.kt | 18 +- .../core/BatchTranscriptionUseCaseTest.kt | 48 +- .../core/MainScreenStateControllerTest.kt | 61 ++- .../core/TranscriptionUiRulesTest.kt | 59 ++- .../SqlDelightAudioCatalogRepositoryTest.kt | 30 +- 29 files changed, 2089 insertions(+), 150 deletions(-) create mode 100644 app/src/androidTest/java/me/maxistar/voiceinbox/AudioShareIntentInstrumentedTest.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/AudioImportCoordinator.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/CatalogRefreshPolicy.kt create mode 100644 app/src/main/java/me/maxistar/voiceinbox/PersistedSelectionAccess.kt create mode 100644 app/src/main/res/xml/imported_audio_paths.xml create mode 100644 app/src/test/java/me/maxistar/voiceinbox/AudioImportCoordinatorTest.kt create mode 100644 app/src/test/java/me/maxistar/voiceinbox/CatalogRefreshPolicyTest.kt create mode 100644 app/src/test/java/me/maxistar/voiceinbox/PersistedSelectionAccessPolicyTest.kt diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/AudioShareIntentInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/AudioShareIntentInstrumentedTest.kt new file mode 100644 index 0000000..148cabd --- /dev/null +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/AudioShareIntentInstrumentedTest.kt @@ -0,0 +1,158 @@ +package me.maxistar.voiceinbox + +import android.content.ClipData +import android.content.Intent +import android.net.Uri +import android.provider.DocumentsContract +import androidx.test.core.app.ActivityScenario +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import me.maxistar.voiceinbox.core.AndroidSqlDelightAudioCatalogFactory +import androidx.work.WorkInfo +import androidx.work.WorkManager +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith +import java.io.File +import java.util.concurrent.TimeUnit + +@RunWith(AndroidJUnit4::class) +class AudioShareIntentInstrumentedTest { + private val context get() = InstrumentationRegistry.getInstrumentation().targetContext + + @Before + fun clearState() { + context.deleteDatabase(AndroidSqlDelightAudioCatalogFactory.DATABASE_NAME) + File(context.filesDir, AndroidAudioImportConstants.DIRECTORY_NAME).deleteRecursively() + WorkManager.getInstance(context) + .cancelUniqueWork(TranscriptionWorker.UNIQUE_WORK_NAME) + .result.get(30, TimeUnit.SECONDS) + } + + @Test + fun parserNormalizesSingleMultipleClipAndMalformedIntents() { + val wav = documentUri("wav") + val m4a = documentUri("m4a") + val single = Intent(Intent.ACTION_SEND) + .putExtra(Intent.EXTRA_STREAM, wav) + assertEquals(listOf(wav), AudioShareIntentParser.streamUris(single)) + + val multiple = Intent(Intent.ACTION_SEND_MULTIPLE) + .putParcelableArrayListExtra(Intent.EXTRA_STREAM, arrayListOf(wav, m4a, wav)) + .apply { clipData = ClipData.newRawUri("audio", m4a) } + assertEquals(listOf(m4a, wav), AudioShareIntentParser.streamUris(multiple)) + assertTrue(AudioShareIntentParser.streamUris(Intent(Intent.ACTION_SEND)).isEmpty()) + assertTrue(AudioShareIntentParser.streamUris(Intent(Intent.ACTION_VIEW)).isEmpty()) + } + + @Test + fun coldWarmAndRecreatedDeliveryCreatesOneDurableImport() { + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { activity -> + MainActivity::class.java.getDeclaredMethod("handleShareIntent", Intent::class.java) + .apply { isAccessible = true } + .invoke(activity, shareIntent(documentUri("wav"))) + } + awaitImportFinished(scenario) + assertEquals(1, importedRows()) + assertTrue(importedDocumentIds().single().startsWith("content://${context.packageName}.files/")) + assertTrue( + WorkManager.getInstance(context) + .getWorkInfosForUniqueWork(TranscriptionWorker.UNIQUE_WORK_NAME) + .get(30, TimeUnit.SECONDS) + .none { it.state in ACTIVE_WORK_STATES }, + ) + + scenario.onActivity { activity -> + MainActivity::class.java.getDeclaredMethod("onNewIntent", Intent::class.java) + .apply { isAccessible = true } + .invoke(activity, shareIntent(documentUri("wav"))) + } + awaitImportFinished(scenario) + assertEquals(1, importedRows()) + } + } + + @Test + fun pickerStyleMultipleInputKeepsAcceptedItemWhenAnotherIsRejected() { + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { activity -> + MainActivity::class.java.getDeclaredMethod("ingestAudioUris", List::class.java) + .apply { isAccessible = true } + .invoke(activity, listOf(documentUri("m4a"), documentUri("text"))) + } + awaitImportFinished(scenario) + assertEquals(1, importedRows()) + } + } + + @Test + fun telegramStyleOggAndOpusMultipleShareImportsBothItems() { + val multiple = Intent(Intent.ACTION_SEND_MULTIPLE) + .setType("audio/*") + .putParcelableArrayListExtra( + Intent.EXTRA_STREAM, + arrayListOf(documentUri("ogg"), documentUri("opus")), + ) + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { activity -> + MainActivity::class.java.getDeclaredMethod("handleShareIntent", Intent::class.java) + .apply { isAccessible = true } + .invoke(activity, multiple) + } + awaitImportFinished(scenario) + assertEquals(2, importedRows()) + } + } + + private fun awaitImportFinished(scenario: ActivityScenario) { + repeat(100) { + var active = true + scenario.onActivity { activity -> + active = MainActivity::class.java.getDeclaredField("ingestionActive") + .apply { isAccessible = true } + .getBoolean(activity) + } + if (!active) return + Thread.sleep(25) + } + error("Timed out waiting for audio ingestion") + } + + private fun importedRows(): Int { + val repository = AndroidSqlDelightAudioCatalogFactory(context).create() + return try { + repository.importedFiles(AndroidAudioImportConstants.SOURCE_ID).size + } finally { + repository.close() + } + } + + private fun importedDocumentIds(): List { + val repository = AndroidSqlDelightAudioCatalogFactory(context).create() + return try { + repository.importedFiles(AndroidAudioImportConstants.SOURCE_ID).map { it.documentUri } + } finally { + repository.close() + } + } + + private fun shareIntent(uri: Uri): Intent = + Intent(context, MainActivity::class.java) + .setAction(Intent.ACTION_SEND) + .setType("audio/wav") + .putExtra(Intent.EXTRA_STREAM, uri) + + private fun documentUri(id: String): Uri = + DocumentsContract.buildDocumentUri(TestAudioDocumentsProvider.AUTHORITY, id) + + private companion object { + val ACTIVE_WORK_STATES = setOf( + WorkInfo.State.ENQUEUED, + WorkInfo.State.BLOCKED, + WorkInfo.State.RUNNING, + ) + } +} diff --git a/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt b/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt index 84e611c..1ef5f65 100644 --- a/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt +++ b/app/src/androidTest/java/me/maxistar/voiceinbox/MainActivityInstrumentedTest.kt @@ -4,6 +4,10 @@ import me.maxistar.voiceinbox.core.* import android.content.Context import android.net.Uri +import android.provider.DocumentsContract +import android.widget.Button +import android.widget.LinearLayout +import android.view.ViewGroup import androidx.test.core.app.ActivityScenario import androidx.test.espresso.Espresso.onView import androidx.test.espresso.Espresso.openActionBarOverflowOrOptionsMenu @@ -14,14 +18,21 @@ import androidx.test.espresso.assertion.ViewAssertions.matches import androidx.test.platform.app.InstrumentationRegistry import androidx.test.espresso.matcher.ViewMatchers.withContentDescription import androidx.test.espresso.matcher.ViewMatchers.isDisplayed +import androidx.test.espresso.matcher.ViewMatchers.isEnabled import androidx.test.espresso.matcher.ViewMatchers.withId import androidx.test.espresso.matcher.ViewMatchers.withText import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.work.ExistingWorkPolicy +import androidx.work.OneTimeWorkRequestBuilder +import androidx.work.WorkManager import org.hamcrest.Matchers.containsString import org.hamcrest.Matchers.not import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue import org.junit.Test import org.junit.runner.RunWith +import java.util.concurrent.TimeUnit @RunWith(AndroidJUnit4::class) class MainActivityInstrumentedTest { @@ -34,6 +45,7 @@ class MainActivityInstrumentedTest { onView(withText(R.string.folder_not_selected)).check(matches(isDisplayed())) onView(withId(R.id.selectOutput)).check(matches(isDisplayed())) onView(withId(R.id.selectFolder)).check(matches(isDisplayed())) + onView(withId(R.id.importAudio)).check(matches(isDisplayed())) onView(withContentDescription(R.string.menu_refresh_folder)).check(matches(isDisplayed())) onView(withId(R.id.transcribeAll)).check(matches(not(isDisplayed()))) @@ -44,6 +56,37 @@ class MainActivityInstrumentedTest { } } + @Test + fun storageSetupIgnoresModelReadinessButRespectsActiveWork() { + clearActivityState() + ActivityScenario.launch(MainActivity::class.java).use { scenario -> + scenario.onActivity { activity -> + MainActivity::class.java.getDeclaredField("modelReady") + .apply { isAccessible = true } + .setBoolean(activity, false) + MainActivity::class.java.getDeclaredField("modelLoading") + .apply { isAccessible = true } + .setBoolean(activity, true) + MainActivity::class.java.getDeclaredField("transcriptionState") + .apply { isAccessible = true } + .set(activity, TranscriptionObservationState.IDLE) + MainActivity::class.java.getDeclaredMethod("updateControls") + .apply { isAccessible = true } + .invoke(activity) + } + + onView(withId(R.id.selectOutput)).check(matches(isEnabled())) + onView(withId(R.id.selectFolder)).check(matches(isEnabled())) + + enqueueDelayedTranscription() + awaitActivity(scenario) { activity -> + transcriptionState(activity) == TranscriptionObservationState.ACTIVE && + !activity.findViewById