From 2d8a2d723c548e312e3672c06bc0a18553a2ff47 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 30 Aug 2026 04:17:03 +0000 Subject: [PATCH 001/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index f5303f8d7..70e568d2c 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6456864568 +DownloadsDownloads6637966379 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index 584248163..bd8be0fc8 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars797797 +StarsStars857857 From b60796adc150de7db47a887b85320a12ad78367f Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 31 Aug 2026 04:22:07 +0000 Subject: [PATCH 002/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index 70e568d2c..8355938fd 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6637966379 +DownloadsDownloads6762067620 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index bd8be0fc8..e13df046c 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars857857 +StarsStars873873 From e49574a7a1b5bdee9d7527a9e07fdb14c3ddc39c Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 1 Sep 2026 04:06:28 +0000 Subject: [PATCH 003/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index 8355938fd..752f8ba83 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6762067620 +DownloadsDownloads6811168111 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index e13df046c..e4d173a7a 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars873873 +StarsStars878878 From c24efc1d6ab5cf10e394c7e2b13607b809a5aa58 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 2 Sep 2026 03:28:38 +0000 Subject: [PATCH 004/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index 752f8ba83..e7457a9b2 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6811168111 +DownloadsDownloads6846068460 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index e4d173a7a..e24ecc9e8 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars878878 +StarsStars885885 From afa02a6889a563c8afa430bb1e4f01ca639f8551 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Sep 2026 03:29:50 +0000 Subject: [PATCH 005/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index e7457a9b2..36bc6b64f 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6846068460 +DownloadsDownloads6873668736 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index e24ecc9e8..1b43d6e3c 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars885885 +StarsStars893893 From f76803773d7e1f60e1f23ec0ce4f67c30def5732 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 4 Sep 2026 03:27:48 +0000 Subject: [PATCH 006/118] chore: update README badges [skip ci] --- docs/badges/downloads.svg | 2 +- docs/badges/stars.svg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/badges/downloads.svg b/docs/badges/downloads.svg index 36bc6b64f..26ac29dd8 100644 --- a/docs/badges/downloads.svg +++ b/docs/badges/downloads.svg @@ -1 +1 @@ -DownloadsDownloads6873668736 +DownloadsDownloads6901869018 diff --git a/docs/badges/stars.svg b/docs/badges/stars.svg index 1b43d6e3c..73142cc70 100644 --- a/docs/badges/stars.svg +++ b/docs/badges/stars.svg @@ -1 +1 @@ -StarsStars893893 +StarsStars895895 From ec749d82a39fcd5015d61054ec3ee28d62a5e525 Mon Sep 17 00:00:00 2001 From: LeanBitLab Date: Fri, 4 Sep 2026 00:41:25 +0530 Subject: [PATCH 007/118] feat(ocr): in-keyboard offline OCR camera viewfinder and screenshot text extraction --- app/build.gradle.kts | 7 + app/src/main/AndroidManifest.xml | 2 + .../keyboard/KeyboardActionListenerImpl.kt | 8 + .../keyboard/keyboard/KeyboardSwitcher.java | 122 ++++++ .../keyboard/internal/KeyboardIconsSet.kt | 3 + .../keyboard_parser/floris/KeyCode.kt | 3 +- .../keyboard/latin/ClipboardHistoryManager.kt | 65 ++- .../keyboard/latin/KeyboardWrapperView.kt | 20 +- .../keyboard/latin/ocr/ITextRecognizer.kt | 15 + .../keyboard/latin/ocr/OcrCameraManager.kt | 203 +++++++++ .../keyboard/latin/ocr/OcrCameraView.kt | 188 +++++++++ .../keyboard/latin/ocr/OcrPipeline.kt | 62 +++ .../keyboard/latin/ocr/OcrPluginLoader.kt | 396 ++++++++++++++++++ .../keyboard/latin/ocr/OcrResultView.kt | 101 +++++ .../keyboard/latin/ocr/ScreenshotHelper.kt | 103 +++++ .../keyboard/latin/utils/ToolbarUtils.kt | 8 +- .../keyboard/settings/SettingsNavHost.kt | 5 + .../preferences/LoadOcrPluginPreference.kt | 255 +++++++++++ .../settings/screens/LibrariesHubScreen.kt | 40 +- .../settings/screens/OcrSettingsScreen.kt | 148 +++++++ app/src/main/res/drawable/ic_flash_off.xml | 10 + app/src/main/res/drawable/ic_flash_on.xml | 10 + app/src/main/res/drawable/ic_ocr.xml | 14 + app/src/main/res/drawable/ic_screenshot.xml | 10 + app/src/main/res/drawable/ic_shutter.xml | 14 + .../main/res/layout/main_keyboard_frame.xml | 6 + app/src/main/res/layout/ocr_camera_view.xml | 128 ++++++ app/src/main/res/layout/ocr_result_view.xml | 86 ++++ .../main/res/layout/screenshot_suggestion.xml | 57 +++ app/src/main/res/values/strings.xml | 32 ++ docs/releasenote/release_notes_v4.1.8.md | 12 +- 31 files changed, 2091 insertions(+), 42 deletions(-) create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/ITextRecognizer.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraManager.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraView.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/OcrPipeline.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/OcrPluginLoader.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/OcrResultView.kt create mode 100644 app/src/main/java/helium314/keyboard/latin/ocr/ScreenshotHelper.kt create mode 100644 app/src/main/java/helium314/keyboard/settings/preferences/LoadOcrPluginPreference.kt create mode 100644 app/src/main/java/helium314/keyboard/settings/screens/OcrSettingsScreen.kt create mode 100644 app/src/main/res/drawable/ic_flash_off.xml create mode 100644 app/src/main/res/drawable/ic_flash_on.xml create mode 100644 app/src/main/res/drawable/ic_ocr.xml create mode 100644 app/src/main/res/drawable/ic_screenshot.xml create mode 100644 app/src/main/res/drawable/ic_shutter.xml create mode 100644 app/src/main/res/layout/ocr_camera_view.xml create mode 100644 app/src/main/res/layout/ocr_result_view.xml create mode 100644 app/src/main/res/layout/screenshot_suggestion.xml diff --git a/app/build.gradle.kts b/app/build.gradle.kts index ad5a090d8..b52d87170 100755 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -251,6 +251,13 @@ dependencies { // Force 16 KB page-aligned version of graphics-path implementation("androidx.graphics:graphics-path:1.1.0") + // CameraX for in-keyboard OCR viewfinder + val cameraxVersion = "1.4.1" + implementation("androidx.camera:camera-core:$cameraxVersion") + implementation("androidx.camera:camera-camera2:$cameraxVersion") + implementation("androidx.camera:camera-lifecycle:$cameraxVersion") + implementation("androidx.camera:camera-view:$cameraxVersion") + // WorkManager — required by plugins loaded via DexClassLoader. // ML Kit internally calls WorkManager.getInstance(context) using the host app context, // so the host app must have WorkManagerInitializer registered in its manifest. diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index ff18a667f..d683e3d71 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -19,7 +19,9 @@ SPDX-License-Identifier: Apache-2.0 AND GPL-3.0-only + + diff --git a/app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListenerImpl.kt b/app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListenerImpl.kt index 23f544e24..2fa5631ff 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListenerImpl.kt +++ b/app/src/main/java/helium314/keyboard/keyboard/KeyboardActionListenerImpl.kt @@ -160,6 +160,14 @@ class KeyboardActionListenerImpl(private val latinIME: LatinIME, private val inp } return } + KeyCode.OCR -> { + if (keyboardSwitcher.isOcrShowing) { + keyboardSwitcher.hideOcrPanels() + } else { + keyboardSwitcher.showOcrCamera() + } + return + } KeyCode.TOGGLE_AUTOCORRECT -> return settings.toggleAutoCorrect() KeyCode.TOGGLE_INCOGNITO_MODE -> { settings.toggleAlwaysIncognitoMode() diff --git a/app/src/main/java/helium314/keyboard/keyboard/KeyboardSwitcher.java b/app/src/main/java/helium314/keyboard/keyboard/KeyboardSwitcher.java index 892122fce..52d1bd4b5 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/KeyboardSwitcher.java +++ b/app/src/main/java/helium314/keyboard/keyboard/KeyboardSwitcher.java @@ -44,6 +44,9 @@ import helium314.keyboard.latin.WordComposer; import helium314.keyboard.latin.handwriting.HandwritingLoader; import helium314.keyboard.latin.handwriting.HandwritingView; +import helium314.keyboard.latin.ocr.OcrCameraView; +import helium314.keyboard.latin.ocr.OcrResultView; +import helium314.keyboard.latin.ocr.OcrPluginLoader; import helium314.keyboard.latin.common.ColorType; import helium314.keyboard.latin.settings.Settings; import helium314.keyboard.latin.settings.SettingsValues; @@ -57,6 +60,7 @@ import helium314.keyboard.latin.utils.ScriptUtils; import helium314.keyboard.latin.utils.SubtypeUtilsAdditional; import helium314.keyboard.latin.utils.ToolbarMode; +import java.util.List; public final class KeyboardSwitcher implements KeyboardState.SwitchActions { private static final String TAG = KeyboardSwitcher.class.getSimpleName(); @@ -73,6 +77,8 @@ public final class KeyboardSwitcher implements KeyboardState.SwitchActions { private LinearLayout mStripContainer; private ClipboardHistoryView mClipboardHistoryView; private HandwritingView mHandwritingView; + private OcrCameraView mOcrCameraView; + private OcrResultView mOcrResultView; private TouchpadView mTouchpadView; private TextView mFakeToastView; private LatinIME mLatinIME; @@ -208,6 +214,9 @@ public void onHideWindow() { if (mKeyboardView != null) { mKeyboardView.onHideWindow(); } + if (mOcrCameraView != null && mOcrCameraView.isShown()) { + mOcrCameraView.stopCamera(); + } } public void onConfigurationChanged(final Configuration newConfig) { @@ -403,6 +412,15 @@ private void setMainKeyboardFrame( } mHandwritingView.setVisibility(View.GONE); } + if (mOcrCameraView != null) { + if (mOcrCameraView.isShown()) { + mOcrCameraView.stopCamera(); + } + mOcrCameraView.setVisibility(View.GONE); + } + if (mOcrResultView != null) { + mOcrResultView.setVisibility(View.GONE); + } if (PointerTracker.sPersistentTouchpadModeActive) { @@ -525,6 +543,71 @@ public void clearHandwritingCanvas() { } } + public void showOcrCamera() { + if (DEBUG_ACTION) { + Log.d(TAG, "showOcrCamera"); + } + PointerTracker.sPersistentTouchpadModeActive = false; + if (mTouchpadView != null) { + mTouchpadView.setVisibility(View.GONE); + } + KeyboardActionListenerImpl.sPersistentTextEditModeActive = false; + mMainKeyboardFrame.setVisibility(View.VISIBLE); + mKeyboardView.setVisibility(View.GONE); + mEmojiTabStripView.setVisibility(View.GONE); + mSuggestionStripView.setVisibility(View.VISIBLE); + mStripContainer.setVisibility(View.VISIBLE); + mClipboardStripScrollView.setVisibility(View.GONE); + mEmojiPalettesView.setVisibility(View.GONE); + mClipboardHistoryView.setVisibility(View.GONE); + if (mHandwritingView != null) { + if (mHandwritingView.isShown()) { + mHandwritingView.stopHandwriting(); + } + mHandwritingView.setVisibility(View.GONE); + } + if (mOcrResultView != null) { + mOcrResultView.setVisibility(View.GONE); + } + if (mOcrCameraView != null) { + mOcrCameraView.setVisibility(View.VISIBLE); + mOcrCameraView.startCamera(); + } + } + + public void showOcrResult(@NonNull final List lines) { + if (DEBUG_ACTION) { + Log.d(TAG, "showOcrResult"); + } + if (mOcrCameraView != null) { + mOcrCameraView.stopCamera(); + mOcrCameraView.setVisibility(View.GONE); + } + if (mOcrResultView != null) { + mOcrResultView.setResultText(lines); + mOcrResultView.applyColors(Settings.getValues().mColors); + mOcrResultView.setVisibility(View.VISIBLE); + } + } + + public void hideOcrPanels() { + if (mOcrCameraView != null) { + if (mOcrCameraView.isShown()) { + mOcrCameraView.stopCamera(); + } + mOcrCameraView.setVisibility(View.GONE); + } + if (mOcrResultView != null) { + mOcrResultView.setVisibility(View.GONE); + } + setAlphabetKeyboard(); + } + + public boolean isOcrShowing() { + return (mOcrCameraView != null && mOcrCameraView.isShown()) + || (mOcrResultView != null && mOcrResultView.isShown()); + } + @Override public void setNumpadKeyboard() { if (DEBUG_ACTION) { @@ -926,6 +1009,10 @@ public void deallocateMemory() { if (mClipboardHistoryView != null) { mClipboardHistoryView.stopClipboardHistory(); } + if (mOcrCameraView != null) { + mOcrCameraView.release(); + } + OcrPluginLoader.release(); } public void trimMemory() { @@ -964,8 +1051,43 @@ public View onCreateInputView(@NonNull Context displayContext, final boolean isH mEmojiPalettesView = mCurrentInputView.findViewById(R.id.emoji_palettes_view); mClipboardHistoryView = mCurrentInputView.findViewById(R.id.clipboard_history_view); mHandwritingView = mCurrentInputView.findViewById(R.id.handwriting_view); + mOcrCameraView = mCurrentInputView.findViewById(R.id.ocr_camera_view); + mOcrResultView = mCurrentInputView.findViewById(R.id.ocr_result_view); mFakeToastView = mCurrentInputView.findViewById(R.id.fakeToast); + if (mOcrCameraView != null) { + mOcrCameraView.setListener(new OcrCameraView.OcrViewListener() { + @Override + public void onOcrTextExtracted(@NonNull List lines) { + showOcrResult(lines); + } + + @Override + public void onCloseOcr() { + hideOcrPanels(); + } + }); + } + if (mOcrResultView != null) { + mOcrResultView.setListener(new OcrResultView.OcrResultListener() { + @Override + public void onInsertText(@NonNull String text) { + mLatinIME.onTextInput(text); + hideOcrPanels(); + } + + @Override + public void onRetake() { + showOcrCamera(); + } + + @Override + public void onClose() { + hideOcrPanels(); + } + }); + } + mKeyboardViewWrapper = mCurrentInputView.findViewById(R.id.keyboard_view_wrapper); mKeyboardViewWrapper.setKeyboardActionListener(mLatinIME.mKeyboardActionListener); mKeyboardView = mCurrentInputView.findViewById(R.id.keyboard_view); diff --git a/app/src/main/java/helium314/keyboard/keyboard/internal/KeyboardIconsSet.kt b/app/src/main/java/helium314/keyboard/keyboard/internal/KeyboardIconsSet.kt index 03014e288..22c3e5c24 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/internal/KeyboardIconsSet.kt +++ b/app/src/main/java/helium314/keyboard/keyboard/internal/KeyboardIconsSet.kt @@ -183,6 +183,7 @@ class KeyboardIconsSet private constructor() { ToolbarKey.SPLIT -> R.drawable.ic_ime_switcher ToolbarKey.PROOFREAD -> R.drawable.ic_proofread ToolbarKey.TRANSLATE -> R.drawable.ic_translate + ToolbarKey.OCR -> R.drawable.ic_ocr ToolbarKey.CUSTOM_AI_1 -> R.drawable.ic_custom_ai_1 ToolbarKey.CUSTOM_AI_2 -> R.drawable.ic_custom_ai_2 ToolbarKey.CUSTOM_AI_3 -> R.drawable.ic_custom_ai_3 @@ -262,6 +263,7 @@ class KeyboardIconsSet private constructor() { ToolbarKey.SPLIT -> R.drawable.ic_ime_switcher ToolbarKey.PROOFREAD -> R.drawable.ic_proofread ToolbarKey.TRANSLATE -> R.drawable.ic_translate + ToolbarKey.OCR -> R.drawable.ic_ocr ToolbarKey.CUSTOM_AI_1 -> R.drawable.ic_custom_ai_1 ToolbarKey.CUSTOM_AI_2 -> R.drawable.ic_custom_ai_2 ToolbarKey.CUSTOM_AI_3 -> R.drawable.ic_custom_ai_3 @@ -341,6 +343,7 @@ class KeyboardIconsSet private constructor() { ToolbarKey.SPLIT -> R.drawable.ic_ime_switcher ToolbarKey.PROOFREAD -> R.drawable.ic_proofread_rounded ToolbarKey.TRANSLATE -> R.drawable.ic_translate_rounded + ToolbarKey.OCR -> R.drawable.ic_ocr ToolbarKey.CUSTOM_AI_1 -> R.drawable.ic_custom_ai_1 ToolbarKey.CUSTOM_AI_2 -> R.drawable.ic_custom_ai_2 ToolbarKey.CUSTOM_AI_3 -> R.drawable.ic_custom_ai_3 diff --git a/app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt b/app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt index 11dd593c8..24c68cd4f 100644 --- a/app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt +++ b/app/src/main/java/helium314/keyboard/keyboard/internal/keyboard_parser/floris/KeyCode.kt @@ -204,6 +204,7 @@ object KeyCode { const val HANDWRITING = -10074 const val CLEAR_HANDWRITING = -10075 const val SWITCH_TO_USER_IME = -10076 + const val OCR = -10077 // Intents @@ -229,7 +230,7 @@ object KeyCode { TIMESTAMP, CTRL_LEFT, CTRL_RIGHT, ALT_LEFT, ALT_RIGHT, META_LEFT, META_RIGHT, SEND_INTENT_ONE, SEND_INTENT_TWO, SEND_INTENT_THREE, INLINE_EMOJI_SEARCH_DONE, META_LOCK, PROOFREAD, TRANSLATE, SHOW_TRANSLATE_LANGUAGES, CUSTOM_AI_1, CUSTOM_AI_2, CUSTOM_AI_3, CUSTOM_AI_4, CUSTOM_AI_5, - CUSTOM_AI_6, CUSTOM_AI_7, CUSTOM_AI_8, CUSTOM_AI_9, CUSTOM_AI_10, CLIPBOARD_SEARCH, TOGGLE_FLOATING_KEYBOARD, TOGGLE_TOUCHPAD_MODE, TOGGLE_TEXT_EDIT_MODE, TOGGLE_SELECTION_MODE, HANDWRITING, CLEAR_HANDWRITING, + CUSTOM_AI_6, CUSTOM_AI_7, CUSTOM_AI_8, CUSTOM_AI_9, CUSTOM_AI_10, CLIPBOARD_SEARCH, TOGGLE_FLOATING_KEYBOARD, TOGGLE_TOUCHPAD_MODE, TOGGLE_TEXT_EDIT_MODE, TOGGLE_SELECTION_MODE, HANDWRITING, CLEAR_HANDWRITING, OCR, CUSTOM1, CUSTOM2, CUSTOM3, CUSTOM4, CUSTOM5, SWITCH_TO_USER_IME -> this diff --git a/app/src/main/java/helium314/keyboard/latin/ClipboardHistoryManager.kt b/app/src/main/java/helium314/keyboard/latin/ClipboardHistoryManager.kt index d5f452355..5fe85fc53 100644 --- a/app/src/main/java/helium314/keyboard/latin/ClipboardHistoryManager.kt +++ b/app/src/main/java/helium314/keyboard/latin/ClipboardHistoryManager.kt @@ -19,12 +19,17 @@ import helium314.keyboard.latin.common.ColorType import helium314.keyboard.latin.common.isValidNumber import helium314.keyboard.latin.database.ClipboardDao import helium314.keyboard.latin.databinding.ClipboardSuggestionBinding +import helium314.keyboard.latin.databinding.ScreenshotSuggestionBinding +import helium314.keyboard.latin.ocr.OcrPluginLoader +import helium314.keyboard.latin.ocr.OcrPipeline +import helium314.keyboard.latin.ocr.ScreenshotHelper import helium314.keyboard.latin.utils.InputTypeUtils import helium314.keyboard.latin.utils.ToolbarKey import android.database.ContentObserver import android.net.Uri import android.os.Handler import android.os.Looper +import android.widget.Toast import kotlin.concurrent.thread import helium314.keyboard.latin.utils.ExecutorUtils import helium314.keyboard.latin.utils.prefs @@ -626,28 +631,68 @@ class ClipboardHistoryManager( lastSuggestedScreenshotUri = contentUri.toString() } - val binding = ClipboardSuggestionBinding.inflate(LayoutInflater.from(latinIME), parent, false) - val textView = binding.clipboardSuggestionText - textView.text = "Screenshot" + val ocrEnabled = OcrPluginLoader.hasPlugin(latinIME) && latinIME.prefs().getBoolean(OcrPluginLoader.PREF_OCR_SUGGEST_SCREENSHOT_TEXT, true) + val binding = ScreenshotSuggestionBinding.inflate(LayoutInflater.from(latinIME), parent, false) + val pasteButton = binding.screenshotPasteButton + val extractButton = binding.screenshotExtractTextButton + val divider = binding.screenshotDivider + val closeButton = binding.screenshotSuggestionClose + + pasteButton.text = latinIME.getString(R.string.ocr_paste_image_button) + extractButton.text = latinIME.getString(R.string.ocr_extract_text_button) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { try { val thumb = latinIME.contentResolver.loadThumbnail(contentUri, android.util.Size(120, 120), null) - val size = Math.min(thumb.width, thumb.height) val x = (thumb.width - size) / 2 val y = (thumb.height - size) / 2 val croppedThumb = android.graphics.Bitmap.createBitmap(thumb, x, y, size, size) - val drawable = android.graphics.drawable.BitmapDrawable(latinIME.resources, croppedThumb) - textView.setCompoundDrawablesRelativeWithIntrinsicBounds(drawable, null, null, null) + pasteButton.setCompoundDrawablesRelativeWithIntrinsicBounds(drawable, null, null, null) } catch (e: Exception) { val clipIcon = latinIME.mKeyboardSwitcher.keyboard.mIconsSet.getIconDrawable(ToolbarKey.PASTE.name.lowercase()) - textView.setCompoundDrawablesRelativeWithIntrinsicBounds(clipIcon, null, null, null) + pasteButton.setCompoundDrawablesRelativeWithIntrinsicBounds(clipIcon, null, null, null) } } - textView.setOnClickListener { + if (ocrEnabled) { + extractButton.visibility = View.VISIBLE + divider.visibility = View.VISIBLE + val ocrIcon = latinIME.mKeyboardSwitcher.keyboard.mIconsSet.getIconDrawable(ToolbarKey.OCR.name.lowercase()) + extractButton.setCompoundDrawablesRelativeWithIntrinsicBounds(ocrIcon, null, null, null) + + extractButton.setOnClickListener { + dontShowCurrentSuggestion = true + lastSuggestedScreenshotUri = contentUri.toString() + AudioAndHapticFeedbackManager.getInstance().performHapticAndAudioFeedback(KeyCode.NOT_SPECIFIED, it, HapticEvent.KEY_PRESS) + binding.root.isGone = true + + ExecutorUtils.getBackgroundExecutor(ExecutorUtils.KEYBOARD).execute { + val bitmap = ScreenshotHelper.loadScaledBitmap(latinIME, contentUri) + if (bitmap != null) { + OcrPipeline(latinIME).processImage( + bitmap = bitmap, + onSuccess = { lines -> + latinIME.mKeyboardSwitcher.showOcrResult(lines) + }, + onError = { err -> + Toast.makeText(latinIME, err, Toast.LENGTH_SHORT).show() + } + ) + } else { + mainHandler.post { + Toast.makeText(latinIME, R.string.ocr_screenshot_load_failed, Toast.LENGTH_SHORT).show() + } + } + } + } + } else { + extractButton.visibility = View.GONE + divider.visibility = View.GONE + } + + pasteButton.setOnClickListener { dontShowCurrentSuggestion = true lastSuggestedScreenshotUri = contentUri.toString() latinIME.onImageSelected(contentUri.toString()) @@ -655,7 +700,6 @@ class ClipboardHistoryManager( binding.root.isGone = true } - val closeButton = binding.clipboardSuggestionClose closeButton.setImageDrawable(latinIME.mKeyboardSwitcher.keyboard.mIconsSet.getIconDrawable(ToolbarKey.CLOSE_HISTORY.name.lowercase())) closeButton.setOnClickListener { val prefs = latinIME.prefs() @@ -671,7 +715,8 @@ class ClipboardHistoryManager( } val colors = latinIME.mSettings.current.mColors - textView.setTextColor(colors.get(ColorType.KEY_TEXT)) + pasteButton.setTextColor(colors.get(ColorType.KEY_TEXT)) + extractButton.setTextColor(colors.get(ColorType.KEY_TEXT)) colors.setColor(closeButton, ColorType.REMOVE_SUGGESTION_ICON) colors.setBackground(binding.root, ColorType.CLIPBOARD_SUGGESTION_BACKGROUND) diff --git a/app/src/main/java/helium314/keyboard/latin/KeyboardWrapperView.kt b/app/src/main/java/helium314/keyboard/latin/KeyboardWrapperView.kt index e4437de03..f604030cc 100644 --- a/app/src/main/java/helium314/keyboard/latin/KeyboardWrapperView.kt +++ b/app/src/main/java/helium314/keyboard/latin/KeyboardWrapperView.kt @@ -134,22 +134,15 @@ class KeyboardWrapperView @JvmOverloads constructor( } override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) { - val keyboardView = findViewById(R.id.keyboard_view) - if (keyboardView == null) { - super.onMeasure(widthMeasureSpec, heightMeasureSpec) - return - } - - super.onMeasure(widthMeasureSpec, heightMeasureSpec) - val settingsValues = Settings.getValues() val keyboardHeight = ResourceUtils.getKeyboardHeight(context.resources, settingsValues) - val padding = keyboardView.paddingTop + keyboardView.paddingBottom + val keyboardView = findViewById(R.id.keyboard_view) + val padding = if (keyboardView != null) keyboardView.paddingTop + keyboardView.paddingBottom else 0 val maxExpectedHeight = keyboardHeight + padding - if (measuredHeight > maxExpectedHeight && maxExpectedHeight > 0) { - setMeasuredDimension(measuredWidth, maxExpectedHeight) - // Re-measure children with the capped height + if (maxExpectedHeight > 0) { + val width = MeasureSpec.getSize(widthMeasureSpec) + setMeasuredDimension(width, maxExpectedHeight) val exactHeightSpec = MeasureSpec.makeMeasureSpec(maxExpectedHeight, MeasureSpec.EXACTLY) for (i in 0 until childCount) { val child = getChildAt(i) @@ -157,7 +150,10 @@ class KeyboardWrapperView @JvmOverloads constructor( measureChildWithMargins(child, widthMeasureSpec, 0, exactHeightSpec, 0) } } + return } + + super.onMeasure(widthMeasureSpec, heightMeasureSpec) } @SuppressLint("RtlHardcoded") diff --git a/app/src/main/java/helium314/keyboard/latin/ocr/ITextRecognizer.kt b/app/src/main/java/helium314/keyboard/latin/ocr/ITextRecognizer.kt new file mode 100644 index 000000000..b2ce3c35b --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/ocr/ITextRecognizer.kt @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.ocr + +import android.content.Context +import android.graphics.Bitmap + +interface ITextRecognizer { + fun getInterfaceVersion(): Int = 1 + fun getScriptName(): String + fun getDisplayName(): String + fun init(context: Context) + fun recognize(bitmap: Bitmap, keepLineBreaks: Boolean): List? + fun isAvailable(): Boolean + fun release() +} diff --git a/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraManager.kt b/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraManager.kt new file mode 100644 index 000000000..565f352a7 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraManager.kt @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.ocr + +import android.annotation.SuppressLint +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Matrix +import android.view.Surface +import androidx.camera.core.Camera +import androidx.camera.core.CameraControl +import androidx.camera.core.CameraSelector +import androidx.camera.core.FocusMeteringAction +import androidx.camera.core.ImageCapture +import androidx.camera.core.ImageCaptureException +import androidx.camera.core.ImageProxy +import androidx.camera.core.Preview +import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import androidx.lifecycle.Lifecycle +import androidx.lifecycle.LifecycleOwner +import androidx.lifecycle.LifecycleRegistry +import helium314.keyboard.latin.utils.Log +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +class OcrCameraManager(private val context: Context) { + private var cameraProvider: ProcessCameraProvider? = null + private var imageCapture: ImageCapture? = null + private var camera: Camera? = null + private var isTorchOn: Boolean = false + private val cameraExecutor: ExecutorService = Executors.newSingleThreadExecutor() + private var lifecycleOwner: ImeLifecycleOwner = ImeLifecycleOwner() + + companion object { + private const val TAG = "OcrCameraManager" + private const val MAX_IMAGE_DIMENSION = 1920 + } + + private class ImeLifecycleOwner : LifecycleOwner { + private val registry = LifecycleRegistry(this) + + init { + registry.currentState = Lifecycle.State.CREATED + } + + fun start() { + registry.currentState = Lifecycle.State.RESUMED + } + + fun stop() { + registry.currentState = Lifecycle.State.CREATED + } + + fun destroy() { + registry.currentState = Lifecycle.State.DESTROYED + } + + override val lifecycle: Lifecycle get() = registry + } + + @SuppressLint("RestrictedApi") + fun startCamera(previewView: PreviewView, onReady: () -> Unit = {}, onError: (Exception) -> Unit = {}) { + val future = ProcessCameraProvider.getInstance(context) + future.addListener({ + try { + cameraProvider = future.get() + bindCamera(previewView) + onReady() + } catch (e: Exception) { + Log.e(TAG, "Failed to start camera", e) + onError(e) + } + }, ContextCompat.getMainExecutor(context)) + } + + private fun bindCamera(previewView: PreviewView) { + val provider = cameraProvider ?: return + provider.unbindAll() + + lifecycleOwner.start() + + val preview = Preview.Builder().build().also { + it.surfaceProvider = previewView.surfaceProvider + } + + imageCapture = ImageCapture.Builder() + .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) + .setTargetRotation(previewView.display?.rotation ?: Surface.ROTATION_0) + .build() + + val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA + + try { + camera = provider.bindToLifecycle( + lifecycleOwner, + cameraSelector, + preview, + imageCapture + ) + isTorchOn = false + } catch (e: Exception) { + Log.e(TAG, "Binding camera use cases failed", e) + } + } + + fun toggleTorch(): Boolean { + val cam = camera ?: return false + return try { + if (cam.cameraInfo.hasFlashUnit()) { + isTorchOn = !isTorchOn + cam.cameraControl.enableTorch(isTorchOn) + isTorchOn + } else { + false + } + } catch (e: Exception) { + Log.e(TAG, "Failed to toggle torch", e) + false + } + } + + fun isTorchEnabled(): Boolean = isTorchOn + + fun capturePhoto(onCaptured: (Bitmap) -> Unit, onError: (Exception) -> Unit) { + val capture = imageCapture ?: run { + onError(IllegalStateException("Camera capture is not ready")) + return + } + + capture.takePicture( + cameraExecutor, + object : ImageCapture.OnImageCapturedCallback() { + override fun onCaptureSuccess(image: ImageProxy) { + try { + val rotation = image.imageInfo.rotationDegrees + val rawBitmap = image.toBitmap() + val scaledBitmap = scaleAndRotateBitmap(rawBitmap, rotation) + if (scaledBitmap != rawBitmap) { + rawBitmap.recycle() + } + image.close() + onCaptured(scaledBitmap) + } catch (e: Exception) { + image.close() + Log.e(TAG, "Error processing captured frame", e) + onError(e) + } + } + + override fun onError(exception: ImageCaptureException) { + Log.e(TAG, "Image capture error", exception) + onError(exception) + } + } + ) + } + + private fun scaleAndRotateBitmap(src: Bitmap, rotationDegrees: Int): Bitmap { + var width = src.width + var height = src.height + + val maxDim = maxOf(width, height) + val scale = if (maxDim > MAX_IMAGE_DIMENSION) { + MAX_IMAGE_DIMENSION.toFloat() / maxDim.toFloat() + } else { + 1.0f + } + + val matrix = Matrix() + if (scale < 1.0f) { + matrix.postScale(scale, scale) + } + if (rotationDegrees != 0) { + matrix.postRotate(rotationDegrees.toFloat()) + } + + return if (!matrix.isIdentity) { + Bitmap.createBitmap(src, 0, 0, width, height, matrix, true) + } else { + src + } + } + + fun stopCamera() { + try { + if (isTorchOn) { + camera?.cameraControl?.enableTorch(false) + isTorchOn = false + } + cameraProvider?.unbindAll() + lifecycleOwner.stop() + } catch (e: Exception) { + Log.e(TAG, "Error stopping camera", e) + } + } + + fun release() { + stopCamera() + lifecycleOwner.destroy() + cameraExecutor.shutdown() + } +} diff --git a/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraView.kt b/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraView.kt new file mode 100644 index 000000000..bca6f82b5 --- /dev/null +++ b/app/src/main/java/helium314/keyboard/latin/ocr/OcrCameraView.kt @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: GPL-3.0-only +package helium314.keyboard.latin.ocr + +import android.content.Context +import android.content.Intent +import android.util.AttributeSet +import android.view.View +import android.widget.Button +import android.widget.FrameLayout +import android.widget.ImageButton +import android.widget.LinearLayout +import android.widget.Toast +import androidx.camera.view.PreviewView +import androidx.core.content.ContextCompat +import helium314.keyboard.latin.R +import helium314.keyboard.latin.common.Colors +import helium314.keyboard.latin.utils.Log +import helium314.keyboard.settings.SettingsActivity +import helium314.keyboard.settings.SettingsDestination + +class OcrCameraView @JvmOverloads constructor( + context: Context, + attrs: AttributeSet? = null, + defStyleAttr: Int = 0 +) : FrameLayout(context, attrs, defStyleAttr) { + + interface OcrViewListener { + fun onOcrTextExtracted(lines: List) + fun onCloseOcr() + } + + private var previewView: PreviewView? = null + private var pluginPanel: LinearLayout? = null + private var controlsBar: LinearLayout? = null + private var shutterBtn: ImageButton? = null + private var flashBtn: ImageButton? = null + private var closeBtn: ImageButton? = null + private var loadingOverlay: FrameLayout? = null + + private var cameraManager: OcrCameraManager? = null + private var pipeline: OcrPipeline? = null + private var listener: OcrViewListener? = null + private var isCameraStarted = false + + companion object { + private const val TAG = "OcrCameraView" + } + + override fun onFinishInflate() { + super.onFinishInflate() + previewView = findViewById(R.id.ocr_preview_view) + pluginPanel = findViewById(R.id.ocr_plugin_required_panel) + controlsBar = findViewById(R.id.ocr_controls_bar) + shutterBtn = findViewById(R.id.btn_ocr_shutter) + flashBtn = findViewById(R.id.btn_ocr_flash) + closeBtn = findViewById(R.id.btn_ocr_close) + loadingOverlay = findViewById(R.id.ocr_loading_overlay) + + previewView?.scaleType = PreviewView.ScaleType.FILL_CENTER + + findViewById