From 58f5ff4d0a24c0e5dd0042367238fe28da57b39a Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 23 Sep 2026 00:35:39 +0000 Subject: [PATCH 1/2] fix(export): stream large JSON downloads to a cache file Small-bucket saves from #229 still buffer the payload as a JS string. For ~500k events that hits the 30s axios timeout and OOMs the WebView. - Intercept /export XHRs and call Android.exportFromUrl - Fetch with auth on a background thread into a cache file - Open the existing Save-to picker from that file CSV of a huge bucket still loads events in JS. Fixes #228 Git-Session-Id: a180614b-5a5a-5d29-84d8-e1c4e076b990 --- .../android/fragments/WebUIFragment.kt | 107 ++++++++++++++++-- .../android/fragments/WebUIFragmentTest.kt | 34 ++++++ 2 files changed, 130 insertions(+), 11 deletions(-) diff --git a/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt b/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt index aea6c122..f064738b 100644 --- a/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt +++ b/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt @@ -151,6 +151,29 @@ internal val ANDROID_EXPORT_HOOK_JS = """ event.stopPropagation(); window.__awAndroidSendBlob(href, el.getAttribute('download') || 'export'); }, true); + + // JSON bucket exports go through axios today. Abort that XHR and stream + // natively so a 30s timeout / JSON.parse of 500k events cannot kill the save. + var xhr = XMLHttpRequest.prototype; + var origOpen = xhr.open; + var origSend = xhr.send; + xhr.open = function (method, url) { + this.__awExportUrl = typeof url === 'string' ? url : ''; + return origOpen.apply(this, arguments); + }; + xhr.send = function () { + var url = this.__awExportUrl || ''; + var match = url.match(/\/0\/(?:buckets\/([^/?#]+)\/)?export(?:[?#]|$)/); + if (match && typeof Android !== 'undefined' && Android.exportFromUrl) { + var filename = match[1] + ? ('aw-bucket-export-' + decodeURIComponent(match[1]) + '.json') + : 'aw-bucket-export.json'; + Android.exportFromUrl(url, filename); + this.abort(); + return; + } + return origSend.apply(this, arguments); + }; })(); """.trimIndent() @@ -253,9 +276,36 @@ internal fun readExportSnapshot(state: Bundle): ExportQueueSnapshot? { } internal fun persistExportPayload(cacheDir: File, content: String): File { + return persistExportStream(cacheDir, content.byteInputStream(StandardCharsets.UTF_8)) +} + +internal fun persistExportStream(cacheDir: File, input: java.io.InputStream): File { val dir = File(cacheDir, "exports").apply { mkdirs() } - return File(dir, "${java.util.UUID.randomUUID()}.export").apply { - writeText(content, StandardCharsets.UTF_8) + val file = File(dir, "${java.util.UUID.randomUUID()}.export") + try { + file.outputStream().use { output -> input.copyTo(output) } + } catch (e: Exception) { + if (file.exists() && !file.delete()) { + Log.w(TAG, "Failed to delete incomplete export cache ${file.name}") + } + throw e + } + return file +} + +internal fun resolveEmbeddedExportUrl(url: String, baseUrl: String = "http://127.0.0.1:5600/"): String? { + val trimmed = url.trim() + if (trimmed.isEmpty()) { + return null + } + return try { + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + trimmed + } else { + URI(baseUrl).resolve(trimmed).toString() + } + } catch (_: Exception) { + null } } @@ -484,7 +534,7 @@ class WebUIFragment : Fragment() { myWebView.settings.javaScriptEnabled = true myWebView.settings.domStorageEnabled = true myWebView.addJavascriptInterface( - WebAppInterface(::queueExport, ::onColorSchemeReported), + WebAppInterface(::queueExport, ::onColorSchemeReported, ::onExportFromUrl), "Android", ) arguments?.let { @@ -531,13 +581,29 @@ class WebUIFragment : Fragment() { } } + private fun onExportFromUrl(url: String, filename: String) { + val base = webView?.url ?: "http://127.0.0.1:5600/" + val resolved = resolveEmbeddedExportUrl(url, base) + if (resolved == null || !isEmbeddedActivityWatchUrl(resolved)) { + Log.w(TAG, "Rejected export URL: $url") + showExportToast(getString(R.string.export_save_failed), long = true) + return + } + downloadEmbeddedExport(resolved, filename, inferExportMimeType(filename, null)) + } + private fun downloadEmbeddedExport(url: String, filename: String, mimeType: String?) { val token = context?.let { ensureDashboardApiKey(it) }.orEmpty() + val cacheDir = context?.applicationContext?.cacheDir ?: return + val safeName = sanitizeExportFilename(filename) + val resolvedMime = inferExportMimeType(safeName, mimeType) thread(name = "aw-export-fetch") { val result = runCatching { val connection = (URL(url).openConnection() as HttpURLConnection).apply { connectTimeout = 15_000 - readTimeout = 60_000 + // Streamed exports send headers immediately, then trickle + // events. Allow long gaps without holding the JSON in RAM. + readTimeout = 120_000 instanceFollowRedirects = true if (token.isNotEmpty()) { setRequestProperty("Authorization", "Bearer $token") @@ -548,15 +614,15 @@ class WebUIFragment : Fragment() { if (code !in 200..299) { error("export HTTP $code") } - connection.inputStream.bufferedReader(StandardCharsets.UTF_8).use { it.readText() } + persistExportStream(cacheDir, connection.inputStream) } finally { connection.disconnect() } } view?.post { result.fold( - onSuccess = { body -> - queueExport(body, filename, inferExportMimeType(filename, mimeType)) + onSuccess = { file -> + queueExportFile(file, safeName, resolvedMime) }, onFailure = { error -> Log.e(TAG, "Failed to fetch export from $url", error) @@ -576,12 +642,25 @@ class WebUIFragment : Fragment() { PendingExport(safeName, resolvedMime, persistExportPayload(cacheDir, content)) } catch (e: Exception) { Log.e(TAG, "Failed to persist export payload", e) - val notify = { - showExportToast(getString(R.string.export_save_failed), long = true) - } - view?.post(notify) ?: if (isAdded) requireActivity().runOnUiThread(notify) else Unit + notifyExportFailed() return } + enqueuePending(pending) + } + + private fun queueExportFile(file: File, filename: String, mimeType: String) { + Log.i(TAG, "Export save requested: $filename ($mimeType, ${file.length()} bytes)") + enqueuePending(PendingExport(filename, mimeType, file)) + } + + private fun notifyExportFailed() { + val notify = { + showExportToast(getString(R.string.export_save_failed), long = true) + } + view?.post(notify) ?: if (isAdded) requireActivity().runOnUiThread(notify) else Unit + } + + private fun enqueuePending(pending: PendingExport) { val enqueue = { if (isAdded) { exportQueue.enqueue(pending) @@ -754,6 +833,7 @@ internal fun writeExport(context: Context, uri: Uri, source: File): Boolean { class WebAppInterface( private val onExport: (content: String, filename: String, mimeType: String) -> Unit, private val onColorScheme: (String) -> Unit = {}, + private val onExportUrl: (url: String, filename: String) -> Unit = { _, _ -> }, ) { private val lock = Any() private val buffer = StringBuilder() @@ -804,4 +884,9 @@ class WebAppInterface( fun reportColorScheme(scheme: String) { onColorScheme(scheme) } + + @JavascriptInterface + fun exportFromUrl(url: String, filename: String) { + onExportUrl(url, filename) + } } diff --git a/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt b/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt index 933ac826..fa6f6f50 100644 --- a/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt +++ b/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt @@ -73,6 +73,8 @@ class WebUIFragmentTest { assertTrue(ANDROID_EXPORT_HOOK_JS.contains("Android.beginExport")) assertTrue(ANDROID_EXPORT_HOOK_JS.contains("Android.appendExport")) assertTrue(ANDROID_EXPORT_HOOK_JS.contains("Android.finishExport")) + assertTrue(ANDROID_EXPORT_HOOK_JS.contains("Android.exportFromUrl")) + assertTrue(ANDROID_EXPORT_HOOK_JS.contains("XMLHttpRequest.prototype")) assertTrue(ANDROID_EXPORT_HOOK_JS.contains("var CHUNK = $EXPORT_BRIDGE_CHUNK_SIZE;")) assertTrue(EXPORT_BRIDGE_CHUNK_SIZE < 1024 * 1024) assertTrue(ANDROID_EXPORT_HOOK_JS.contains("/\\.csv$/i")) @@ -231,6 +233,38 @@ class WebUIFragmentTest { assertEquals("{\"ok\":true}", file.readText()) } + @Test + fun `persistExportStream copies bytes without holding a string`() { + val payload = ByteArray(64 * 1024) { it.toByte() } + val file = persistExportStream(createTempDir(), payload.inputStream()) + assertTrue(file.isFile) + assertEquals(payload.toList(), file.readBytes().toList()) + } + + @Test + fun `resolveEmbeddedExportUrl accepts loopback and relative API paths`() { + assertEquals( + "http://127.0.0.1:5600/api/0/export", + resolveEmbeddedExportUrl("/api/0/export", "http://127.0.0.1:5600/#/buckets"), + ) + assertEquals( + "http://127.0.0.1:5600/api/0/buckets/aw-watcher/export", + resolveEmbeddedExportUrl("http://127.0.0.1:5600/api/0/buckets/aw-watcher/export"), + ) + assertNull(resolveEmbeddedExportUrl(" ")) + } + + @Test + fun `WebAppInterface exportFromUrl reaches the native downloader`() { + val received = mutableListOf>() + val bridge = WebAppInterface( + onExport = { _, _, _ -> error("string export must not run") }, + onExportUrl = { url, filename -> received.add(url to filename) }, + ) + bridge.exportFromUrl("/api/0/export", "aw-bucket-export.json") + assertEquals(listOf("/api/0/export" to "aw-bucket-export.json"), received) + } + private fun cachedExport(dir: File, name: String, content: String): PendingExport { return PendingExport(name, "application/json", File(dir, name).also { it.writeText(content) }) } From 2a39554fb1afa90b962b6e75120749261149ca6c Mon Sep 17 00:00:00 2001 From: Bob Date: Wed, 23 Sep 2026 00:52:24 +0000 Subject: [PATCH 2/2] fix(export): marshal JS-bridge export onto the UI thread JavaScript-interface callbacks run on the WebView bridge thread, so reading webView.url there can throw before the native download starts. A detached view also drops view?.post, leaking the streamed cache file. Post both the URL resolve and the fetch completion through the main-looper Handler; enqueue if the fragment is still added, otherwise delete the file. Git-Session-Id: a434bdbc-0d89-5467-a891-085a8060b1c3 --- .../android/fragments/WebUIFragment.kt | 72 +++++++++++-------- .../android/fragments/WebUIFragmentTest.kt | 9 +++ 2 files changed, 51 insertions(+), 30 deletions(-) diff --git a/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt b/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt index f064738b..2132dfe2 100644 --- a/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt +++ b/mobile/src/main/java/net/activitywatch/android/fragments/WebUIFragment.kt @@ -549,6 +549,22 @@ class WebUIFragment : Fragment() { reloadHandler.postDelayed(reloadRunnable, delay) } + /** + * Always run [action] on the main looper. + * + * JS-bridge callbacks and background fetch completions must not use + * `view?.post`: a detached view drops the runnable, so a finished export + * is never queued and the cache file leaks. The fragment Handler still + * fires after the view is gone; callers then enqueue or delete. + */ + private fun postToUi(action: () -> Unit) { + if (Looper.myLooper() == Looper.getMainLooper()) { + action() + } else { + reloadHandler.post(action) + } + } + override fun onDestroyView() { reloadHandler.removeCallbacks(reloadRunnable) filePathCallback?.onReceiveValue(null) @@ -582,14 +598,19 @@ class WebUIFragment : Fragment() { } private fun onExportFromUrl(url: String, filename: String) { - val base = webView?.url ?: "http://127.0.0.1:5600/" - val resolved = resolveEmbeddedExportUrl(url, base) - if (resolved == null || !isEmbeddedActivityWatchUrl(resolved)) { - Log.w(TAG, "Rejected export URL: $url") - showExportToast(getString(R.string.export_save_failed), long = true) - return + // @JavascriptInterface runs on the WebView bridge thread. webView.url + // (and Toast) are UI-thread only. + postToUi { + if (!isAdded) return@postToUi + val base = webView?.url ?: "http://127.0.0.1:5600/" + val resolved = resolveEmbeddedExportUrl(url, base) + if (resolved == null || !isEmbeddedActivityWatchUrl(resolved)) { + Log.w(TAG, "Rejected export URL: $url") + showExportToast(getString(R.string.export_save_failed), long = true) + return@postToUi + } + downloadEmbeddedExport(resolved, filename, inferExportMimeType(filename, null)) } - downloadEmbeddedExport(resolved, filename, inferExportMimeType(filename, null)) } private fun downloadEmbeddedExport(url: String, filename: String, mimeType: String?) { @@ -619,17 +640,15 @@ class WebUIFragment : Fragment() { connection.disconnect() } } - view?.post { - result.fold( - onSuccess = { file -> - queueExportFile(file, safeName, resolvedMime) - }, - onFailure = { error -> - Log.e(TAG, "Failed to fetch export from $url", error) - showExportToast(getString(R.string.export_save_failed), long = true) - }, - ) - } + result.fold( + onSuccess = { file -> + queueExportFile(file, safeName, resolvedMime) + }, + onFailure = { error -> + Log.e(TAG, "Failed to fetch export from $url", error) + notifyExportFailed() + }, + ) } } @@ -654,14 +673,15 @@ class WebUIFragment : Fragment() { } private fun notifyExportFailed() { - val notify = { - showExportToast(getString(R.string.export_save_failed), long = true) + postToUi { + if (isAdded) { + showExportToast(getString(R.string.export_save_failed), long = true) + } } - view?.post(notify) ?: if (isAdded) requireActivity().runOnUiThread(notify) else Unit } private fun enqueuePending(pending: PendingExport) { - val enqueue = { + postToUi { if (isAdded) { exportQueue.enqueue(pending) launchNextExportPicker() @@ -669,14 +689,6 @@ class WebUIFragment : Fragment() { pending.deleteCache() } } - val view = view - if (view != null) { - view.post(enqueue) - } else if (isAdded) { - requireActivity().runOnUiThread(enqueue) - } else { - pending.deleteCache() - } } private fun launchNextExportPicker() { diff --git a/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt b/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt index fa6f6f50..196df51c 100644 --- a/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt +++ b/mobile/src/test/java/net/activitywatch/android/fragments/WebUIFragmentTest.kt @@ -241,6 +241,15 @@ class WebUIFragmentTest { assertEquals(payload.toList(), file.readBytes().toList()) } + @Test + fun `streamed export cache is deleted when it can no longer be queued`() { + val file = persistExportStream(createTempDir(), "payload".byteInputStream()) + val pending = PendingExport("orphan.json", "application/json", file) + assertTrue(pending.cacheFile.isFile) + pending.deleteCache() + assertFalse(pending.cacheFile.exists()) + } + @Test fun `resolveEmbeddedExportUrl accepts loopback and relative API paths`() { assertEquals(