Fix device fixture allocation and preserve per-case diagnostics

app/build.gradle.kts

@@ -55,6 +55,9 @@ android {
         include("*.schema.json")
     }
     sourceSets.getByName("test").resources.srcDir(rootProject.file("config-schema"))
+    // Device-test lifecycle logic also runs as deterministic JVM tests.
+    sourceSets.getByName("test").java.srcDir("src/sharedTest/java")
+    sourceSets.getByName("androidTest").java.srcDir("src/sharedTest/java")
 
     defaultConfig {
         applicationId = "net.megaproxy487"

app/src/androidTest/java/net/megaproxy487/DeviceTestBase.kt

@@ -26,10 +26,8 @@ import net.megaproxy487.vpn.VpnConnectionState
 import net.megaproxy487.vpn.VpnRuntimeState
 import org.junit.Assert.*
 import org.junit.Rule
-import org.junit.rules.ExternalResource
-import org.junit.rules.RuleChain
-import org.junit.rules.TestWatcher
-import org.junit.runner.Description
+import java.util.concurrent.ThreadPoolExecutor
+import java.util.concurrent.TimeUnit
 import java.io.File
 import java.net.InetAddress
 import java.net.InetSocketAddress
@@ -159,23 +157,27 @@ abstract class DeviceTestBase {
         }.isSuccess)
     }
 
-    private val platform = object : ExternalResource() {
-        override fun before() {
-            listOf("png", "xml").forEach { File(context.getExternalFilesDir(null), "device-failure.$it").delete() }
-            if (arguments.getString("phase") != "verify") io {
-                assertTrue(context.getSharedPreferences("proxy_config", Context.MODE_PRIVATE).edit().clear().commit())
-                val profile = store.activeProfile()
-                store.saveProfile(profile.copy(name = "Device fixture", config = ProxyConfig(
-                    host = "10.0.2.2", port = argument("proxyPort").toInt(), username = "exit",
-                    password = argument("proxyPassword"), allowInvalidProxyCertificate = true,
-                )))
-                store.saveGlobalConnectionSettings(GlobalConnectionSettings(bypassLocalNetworks = false))
-            }
-            assertTrue(context.getSharedPreferences("battery_optimization_reminder", Context.MODE_PRIVATE)
-                .edit().putLong("last_request_at", System.currentTimeMillis()).commit())
-            scenario = ActivityScenario.launch(MainActivity::class.java)
-            device.waitForIdle()
+    private fun prepare() {
+        io {
+            PersistentDiagnosticLog.initialize(context, PersistentDiagnosticLog.DEFAULT_LIMIT_MB)
+            PersistentDiagnosticLog.clear()
+            flushDiagnosticLog()
+            PersistentDiagnosticLog.write("event=device_test stage=start")
+        }
+        listOf("png", "xml").forEach { File(context.getExternalFilesDir(null), "device-failure.$it").delete() }
+        if (arguments.getString("phase") != "verify") io {
+            assertTrue(context.getSharedPreferences("proxy_config", Context.MODE_PRIVATE).edit().clear().commit())
+            val profile = store.activeProfile()
+            store.saveProfile(profile.copy(name = "Device fixture", config = ProxyConfig(
+                host = "10.0.2.2", port = argument("proxyPort").toInt(), username = "exit",
+                password = argument("proxyPassword"), allowInvalidProxyCertificate = true,
+            )))
+            store.saveGlobalConnectionSettings(GlobalConnectionSettings(bypassLocalNetworks = false))
         }
+        assertTrue(context.getSharedPreferences("battery_optimization_reminder", Context.MODE_PRIVATE)
+            .edit().putLong("last_request_at", System.currentTimeMillis()).commit())
+        scenario = ActivityScenario.launch(MainActivity::class.java)
+        device.waitForIdle()
     }
     private fun cleanup() {
         repeat(3) {
@@ -191,21 +193,31 @@ abstract class DeviceTestBase {
             saved()
         }
     }
-    private val evidence = object : TestWatcher() {
-        override fun failed(error: Throwable, description: Description) {
-            // The disposable fixture uses synthetic credentials. This is the app's
-            // sanitized, bounded log, including recovery events absent from logcat.
-            println("MegaProxy diagnostic tail:\n${PersistentDiagnosticLog.readTail(32 * 1024)}")
-            val directory = context.getExternalFilesDir(null)!!
-            device.takeScreenshot(File(directory, "device-failure.png"))
-            device.dumpWindowHierarchy(File(directory, "device-failure.xml"))
-        }
-    }
-    private val activity = object : ExternalResource() {
-        override fun after() { scenario.close() }
+    private fun captureFailure(error: Throwable) {
+        println("Device failure: ${error.javaClass.simpleName}")
+        val directory = requireNotNull(context.getExternalFilesDir(null))
+        device.takeScreenshot(File(directory, "device-failure.png"))
+        device.dumpWindowHierarchy(File(directory, "device-failure.xml"))
     }
-    private val finish = object : ExternalResource() {
-        override fun after() { cleanup() }
+
+    private fun flushDiagnosticLog() {
+        // Test-only barrier after VPN shutdown; a busy/discarded barrier fails with a timeout.
+        val field = PersistentDiagnosticLog::class.java.getDeclaredField("executor").apply { isAccessible = true }
+        val executor = field.get(null) as ThreadPoolExecutor
+        executor.submit {}.get(5, TimeUnit.SECONDS)
     }
-    @get:Rule val rules: RuleChain = RuleChain.outerRule(platform).around(activity).around(finish).around(evidence)
+
+    @get:Rule val rules: org.junit.rules.TestRule = DeviceTestEvidenceRule(
+        prepare = ::prepare,
+        cleanup = { if (::scenario.isInitialized) cleanup() },
+        close = { if (::scenario.isInitialized) scenario.close() },
+        evidence = ::captureFailure,
+        diagnostics = {
+            io {
+                PersistentDiagnosticLog.write("event=device_test stage=finish")
+                flushDiagnosticLog()
+            }
+            println("MegaProxy diagnostic tail:\n${PersistentDiagnosticLog.readTail(64 * 1024)}")
+        },
+    )
 }

app/src/sharedTest/java/net/megaproxy487/DeviceTestEvidenceRule.kt

@@ -0,0 +1,40 @@
+package net.megaproxy487
+
+import org.junit.AssumptionViolatedException
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.MultipleFailureException
+import org.junit.runners.model.Statement
+
+/** Test-only lifecycle shared by instrumentation and plain JVM regression tests. */
+internal class DeviceTestEvidenceRule(
+    private val prepare: () -> Unit,
+    private val cleanup: () -> Unit,
+    private val close: () -> Unit,
+    private val evidence: (Throwable) -> Unit,
+    private val diagnostics: () -> Unit,
+) : TestRule {
+    override fun apply(base: Statement, description: Description) = object : Statement() {
+        override fun evaluate() {
+            val errors = mutableListOf<Throwable>()
+            var captured = false
+            fun attempt(action: () -> Unit) {
+                try {
+                    action()
+                } catch (error: Throwable) {
+                    errors.add(error)
+                    if (error !is AssumptionViolatedException && !captured) {
+                        captured = true
+                        try { evidence(error) } catch (captureError: Throwable) { errors.add(captureError) }
+                    }
+                }
+            }
+            attempt(prepare)
+            if (errors.isEmpty()) attempt { base.evaluate() }
+            attempt(cleanup)
+            attempt(close)
+            attempt(diagnostics)
+            MultipleFailureException.assertEmpty(errors)
+        }
+    }
+}

app/src/test/java/net/megaproxy487/DeviceTestEvidenceRuleTest.kt

@@ -0,0 +1,60 @@
+package net.megaproxy487
+
+import org.junit.Assert.*
+import org.junit.AssumptionViolatedException
+import org.junit.Test
+import org.junit.runner.Description
+import org.junit.runners.model.MultipleFailureException
+import org.junit.runners.model.Statement
+
+class DeviceTestEvidenceRuleTest {
+    private fun run(events: MutableList<String>, failures: Map<String, Throwable> = emptyMap()) {
+        fun step(name: String) {
+            events.add(name)
+            failures[name]?.let { throw it }
+        }
+        DeviceTestEvidenceRule(
+            prepare = { step("prepare") }, cleanup = { step("cleanup") },
+            close = { step("close") }, evidence = { step("evidence") },
+            diagnostics = { step("diagnostics") },
+        ).apply(object : Statement() {
+            override fun evaluate() = step("body")
+        }, Description.EMPTY).evaluate()
+    }
+
+    @Test fun successStillCollectsDiagnostics() {
+        val events = mutableListOf<String>()
+        run(events)
+        assertEquals(listOf("prepare", "body", "cleanup", "close", "diagnostics"), events)
+    }
+
+    @Test fun eachFailureIsCapturedBeforeTheNextPhaseAndStillRunsCleanup() {
+        for (phase in listOf("prepare", "body", "cleanup", "close", "diagnostics")) {
+            val events = mutableListOf<String>()
+            val failure = IllegalStateException(phase)
+            val thrown = runCatching { run(events, mapOf(phase to failure)) }.exceptionOrNull()
+            assertSame(phase, failure, thrown)
+            val expected = mutableListOf("prepare", "body", "cleanup", "close", "diagnostics")
+            if (phase == "prepare") expected.remove("body")
+            expected.add(expected.indexOf(phase) + 1, "evidence")
+            assertEquals(phase, expected, events)
+        }
+    }
+
+    @Test fun evidenceAndCleanupFailuresDoNotHideTheOriginalFailure() {
+        val events = mutableListOf<String>()
+        val failures = listOf("body", "evidence", "cleanup", "close", "diagnostics")
+            .associateWith { IllegalStateException(it) }
+        val thrown = runCatching { run(events, failures) }.exceptionOrNull()
+        assertTrue(thrown is MultipleFailureException)
+        assertEquals(failures.values.toList(), (thrown as MultipleFailureException).failures)
+        assertEquals(1, events.count { it == "evidence" })
+    }
+
+    @Test fun skippedBodyStillCleansUpWithoutFailureEvidence() {
+        val events = mutableListOf<String>()
+        val skipped = AssumptionViolatedException("unsupported API")
+        assertSame(skipped, runCatching { run(events, mapOf("body" to skipped)) }.exceptionOrNull())
+        assertEquals(listOf("prepare", "body", "cleanup", "close", "diagnostics"), events)
+    }
+}

docs/en/release-testing.md

@@ -59,6 +59,8 @@ physical network handover, OEM power policy, every Always-on/lockdown mode or re
 
 The disposable emulator disables public captive-portal probes, reconnects Wi-Fi on API 35 (the API 26 `svc wifi` client exits with 137) and waits for it to become the validated default network. Before starting VPN scenarios it must receive the local SSH fixture banner through `10.0.2.2`; failure saves network diagnostics and stops the suite without retrying tests.
 
+Each case waits for the previous app process and activities to finish, then reads the precise device timestamp of its unique Logcat start marker, without clearing global buffers. Success and failure outputs include a sanitized diagnostic tail (up to 64 KiB), collected after VPN cleanup. Setup, body and cleanup failures retain failure evidence; evidence collection cannot hide the original error. The local TCP/UDP fixture retries only port-allocation collisions, before starting containers or tests.
+
 ## Release acceptance: smoke first, expand by risk
 
 Identify the exact candidate commit, version/ABI, APK SHA-256 and signer. Use full successful CI for

docs/ru/release-testing.md

@@ -60,6 +60,8 @@ instrumentation-процессом; Keystore seed/verify должны остав
 
 Одноразовый эмулятор отключает публичные проверки captive portal, переподключает Wi-Fi на API 35 (клиент `svc wifi` API 26 завершается с кодом 137) и ждёт, пока она станет подтверждённой сетью по умолчанию. До VPN-сценариев он должен получить баннер локального SSH-стенда через `10.0.2.2`; при отказе сохраняется диагностика сети, а набор останавливается без повторения тестов.
 
+Перед каждым сценарием runner ждёт завершения предыдущего процесса и Activity, читает точное время собственной уникальной метки в Logcat, не очищая общие буферы. Вывод успешных и упавших сценариев содержит обезличенный хвост диагностического журнала (до 64 KiB), собранный после остановки VPN. Ошибки подготовки, самого теста и cleanup сохраняют доказательства; ошибка их сбора не скрывает исходную. Локальная TCP/UDP-фикстура повторяет только выбор занятого порта, до запуска контейнеров и тестов.
+
 ## Приёмка релиза: smoke, затем риски изменения
 
 Зафиксируйте точный commit, версию/ABI, SHA-256 и подпись APK. Нужен полный успешный CI кандидата;