diff --git a/.skills/compose-ui/strings-index.txt b/.skills/compose-ui/strings-index.txt index ed13dd5a814..f34e47a65e0 100644 --- a/.skills/compose-ui/strings-index.txt +++ b/.skills/compose-ui/strings-index.txt @@ -341,6 +341,7 @@ debug_search_prev debug_store_logs_summary debug_store_logs_title debug_tab_app_logs +debug_tab_mirror debug_tab_packets deep_link_connect_message deep_link_connect_title @@ -391,6 +392,17 @@ device_metrics_log device_metrics_numeric_value device_metrics_percent_value device_metrics_voltage_value +device_model_capture +device_model_capture_failed +device_model_capture_saved +device_model_load_failed +device_model_rest_face_down +device_model_rest_face_up +device_model_rest_left_side +device_model_rest_right_side +device_model_rest_upright +device_model_view +device_model_view_in_space device_sleeping device_storage_ui_title device_theme_language @@ -1177,6 +1189,24 @@ metric_channel_label micrograms_per_cubic_meter min minimum_wake_time_seconds +### MIRROR ### +mirror_active +mirror_back +mirror_device_screen +mirror_eink_hint +mirror_frame_info +mirror_hint_keyboard_active +mirror_hint_tap_keyboard +mirror_hint_tap_touch +mirror_key_down +mirror_key_left +mirror_key_ok +mirror_key_right +mirror_key_up +mirror_keyboard +mirror_no_frame +mirror_not_connected +mirror_off modem_preset module_settings modules_already_unlocked diff --git a/androidApp/build.gradle.kts b/androidApp/build.gradle.kts index 0e4ed76cf46..f6f1bce6bed 100644 --- a/androidApp/build.gradle.kts +++ b/androidApp/build.gradle.kts @@ -270,6 +270,10 @@ dependencies { googleImplementation(libs.location.services) googleImplementation(libs.play.services.maps) + // ARCore for the device model viewer; proprietary, so google only. Same exclusions as feature/node's sceneview. + googleImplementation("io.github.sceneview:arsceneview:${libs.versions.sceneview.get()}") { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-android-extensions-runtime") + } googleImplementation(libs.maps.compose) googleImplementation(libs.maps.compose.utils) googleImplementation(libs.maps.compose.widgets) diff --git a/androidApp/src/fdroid/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt b/androidApp/src/fdroid/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt new file mode 100644 index 00000000000..f89456e2a5f --- /dev/null +++ b/androidApp/src/fdroid/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.app.node + +import org.meshtastic.core.ui.util.DeviceModelArViewer + +/** ARCore is proprietary, so the fdroid flavor has no AR viewer and the action never shows. */ +val deviceModelArViewer: DeviceModelArViewer? = null diff --git a/androidApp/src/google/kotlin/org/meshtastic/app/node/ArCapture.kt b/androidApp/src/google/kotlin/org/meshtastic/app/node/ArCapture.kt new file mode 100644 index 00000000000..54aa3a2fdd7 --- /dev/null +++ b/androidApp/src/google/kotlin/org/meshtastic/app/node/ArCapture.kt @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.app.node + +import android.content.ContentValues +import android.content.Context +import android.graphics.Bitmap +import android.net.Uri +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.provider.MediaStore +import android.view.PixelCopy +import android.view.SurfaceView +import android.view.View +import android.view.ViewGroup +import androidx.annotation.RequiresApi +import androidx.core.view.children +import kotlinx.coroutines.CoroutineDispatcher +import kotlinx.coroutines.suspendCancellableCoroutine +import kotlinx.coroutines.withContext +import java.text.SimpleDateFormat +import java.util.Date +import java.util.Locale +import kotlin.coroutines.resume + +private const val PICTURES_RELATIVE_PATH = "Pictures/Meshtastic" +private const val JPEG_QUALITY = 92 + +/** + * Copies the AR view's GL surface into Pictures/Meshtastic: the camera feed and the model, none of the Compose controls + * drawn over it. The scene renders every camera frame, so the surface is never stale. Null when nothing was saved. + */ +@RequiresApi(Build.VERSION_CODES.Q) +internal suspend fun captureArScene(root: View, title: String, io: CoroutineDispatcher): Uri? { + val surface = root.findSurfaceView()?.takeIf { it.width > 0 && it.height > 0 } ?: return null + val bitmap = Bitmap.createBitmap(surface.width, surface.height, Bitmap.Config.ARGB_8888) + val copied = suspendCancellableCoroutine { continuation -> + PixelCopy.request( + surface, + bitmap, + { result -> continuation.resume(result == PixelCopy.SUCCESS) }, + Handler(Looper.getMainLooper()), + ) + } + return if (copied) withContext(io) { saveToPictures(root.context, bitmap, title) } else null +} + +private fun View.findSurfaceView(): SurfaceView? = when (this) { + is SurfaceView -> this + is ViewGroup -> children.firstNotNullOfOrNull { it.findSurfaceView() } + else -> null +} + +@RequiresApi(Build.VERSION_CODES.Q) +private fun saveToPictures(context: Context, bitmap: Bitmap, title: String): Uri? { + val resolver = context.contentResolver + val stamp = SimpleDateFormat("yyyyMMdd-HHmmss", Locale.US).format(Date()) + val name = "${title.replace(Regex("[^A-Za-z0-9]+"), "-").trim('-')}-$stamp.jpg" + val pending = + ContentValues().apply { + put(MediaStore.Images.Media.DISPLAY_NAME, name) + put(MediaStore.Images.Media.MIME_TYPE, "image/jpeg") + put(MediaStore.Images.Media.RELATIVE_PATH, PICTURES_RELATIVE_PATH) + put(MediaStore.Images.Media.IS_PENDING, 1) + } + val uri = resolver.insert(MediaStore.Images.Media.getContentUri(MediaStore.VOLUME_EXTERNAL_PRIMARY), pending) + val written = + uri != null && + runCatching { + resolver.openOutputStream(uri)?.use { bitmap.compress(Bitmap.CompressFormat.JPEG, JPEG_QUALITY, it) } + } + .getOrNull() == true + // A row left pending would sit hidden in the gallery, so a failed write removes it. + if (written) { + resolver.update(uri, ContentValues().apply { put(MediaStore.Images.Media.IS_PENDING, 0) }, null, null) + } else if (uri != null) { + resolver.delete(uri, null, null) + } + return uri.takeIf { written } +} diff --git a/androidApp/src/google/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt b/androidApp/src/google/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt new file mode 100644 index 00000000000..5e4dbfb89c7 --- /dev/null +++ b/androidApp/src/google/kotlin/org/meshtastic/app/node/DeviceModelArViewer.kt @@ -0,0 +1,441 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.app.node + +import android.Manifest +import android.app.Activity +import android.content.Context +import android.content.ContextWrapper +import android.content.pm.ActivityInfo +import android.content.pm.PackageManager +import android.os.Build +import android.widget.Toast +import androidx.activity.compose.BackHandler +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.foundation.background +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.safeDrawingPadding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.FilledIconButton +import androidx.compose.material3.FilterChip +import androidx.compose.material3.FilterChipDefaults +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.SideEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.key +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.hapticfeedback.HapticFeedbackType +import androidx.compose.ui.layout.onSizeChanged +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalHapticFeedback +import androidx.compose.ui.platform.LocalView +import androidx.compose.ui.unit.IntSize +import androidx.compose.ui.unit.dp +import androidx.core.content.ContextCompat +import com.google.ar.core.Anchor +import com.google.ar.core.ArCoreApk +import com.google.ar.core.Config +import com.google.ar.core.Frame +import com.google.ar.core.Plane +import com.google.ar.core.Session +import com.google.ar.core.TrackingFailureReason +import com.google.ar.core.TrackingState +import io.github.sceneview.ar.ARCoachingOverlay +import io.github.sceneview.ar.ARSceneView +import io.github.sceneview.ar.PlacementReticleVisual +import io.github.sceneview.ar.ReticlePhase +import io.github.sceneview.ar.arcore.rememberDetectedPlanes +import io.github.sceneview.ar.placementHit +import io.github.sceneview.ar.rememberArGuidanceState +import io.github.sceneview.ar.reticlePhaseFor +import io.github.sceneview.ar.scene.SceneUnderstanding +import io.github.sceneview.math.Position +import io.github.sceneview.math.Rotation +import io.github.sceneview.model.model +import io.github.sceneview.rememberEngine +import io.github.sceneview.rememberMaterialLoader +import io.github.sceneview.rememberModelLoader +import io.github.sceneview.rememberOnGestureListener +import kotlinx.coroutines.launch +import org.jetbrains.compose.resources.StringResource +import org.jetbrains.compose.resources.stringResource +import org.koin.compose.koinInject +import org.meshtastic.core.di.CoroutineDispatchers +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.close +import org.meshtastic.core.resources.device_model_capture +import org.meshtastic.core.resources.device_model_capture_failed +import org.meshtastic.core.resources.device_model_capture_saved +import org.meshtastic.core.resources.device_model_rest_face_down +import org.meshtastic.core.resources.device_model_rest_face_up +import org.meshtastic.core.resources.device_model_rest_left_side +import org.meshtastic.core.resources.device_model_rest_right_side +import org.meshtastic.core.resources.device_model_rest_upright +import org.meshtastic.core.resources.info +import org.meshtastic.core.ui.icon.Close +import org.meshtastic.core.ui.icon.Info +import org.meshtastic.core.ui.icon.MeshtasticIcons +import org.meshtastic.core.ui.icon.PhotoCamera +import org.meshtastic.core.ui.util.DeviceModelArViewer +import org.meshtastic.feature.node.component.DeviceScreenNode +import org.meshtastic.feature.node.model.GlbBounds +import org.meshtastic.feature.node.model.RestPose +import java.nio.ByteBuffer + +private const val SCRIM_ALPHA = 0.4f +private const val YAW_DEGREES_PER_PX = 0.4f + +val deviceModelArViewer: DeviceModelArViewer? = GoogleDeviceModelArViewer + +/** + * The google flavor's AR viewer. A phone ARCore is still checking, or that only lacks the ARCore app, counts as + * available: the viewer's own overlay offers the install. + */ +private object GoogleDeviceModelArViewer : DeviceModelArViewer { + @Composable + override fun isAvailable(): Boolean { + val context = LocalContext.current + return remember(context) { !ArCoreApk.getInstance().checkAvailability(context).isUnsupported } + } + + @Composable + override fun Content( + glb: ByteArray, + title: String, + notice: String?, + onDismiss: () -> Unit, + screenMin: FloatArray?, + screenMax: FloatArray?, + screenFrame: MirrorFrame?, + ) = DeviceModelAr(glb, title, notice, onDismiss, ArScreen(screenMin, screenMax, screenFrame)) +} + +/** + * Tap a surface to stand the device on it at real size: the models are in meters and ARCore works in meters, so nothing + * is scaled. A later tap moves it rather than adding another, and a twist turns it. + */ +@Composable +private fun DeviceModelAr(glb: ByteArray, title: String, notice: String?, onDismiss: () -> Unit, screen: ArScreen) { + BackHandler(onBack = onDismiss) + LockOrientationWhileShown() + // Asked here rather than left to the AR view, whose own request never appears from inside a dialog. + val context = LocalContext.current + var cameraGranted by remember { + mutableStateOf( + ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED, + ) + } + val requestCamera = + rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted -> + if (granted) cameraGranted = true else onDismiss() + } + var asked by rememberSaveable { mutableStateOf(false) } + SideEffect { + if (!cameraGranted && !asked) { + asked = true + requestCamera.launch(Manifest.permission.CAMERA) + } + } + Box(modifier = Modifier.fillMaxSize()) { + if (cameraGranted) DeviceModelArScene(glb, screen, onCapture = rememberArCapture(title)) + ArHeader(title, notice, onDismiss) + } +} + +/** + * ARSceneView rather than SceneView's PlacementScene, which as of 4.56.0 does not pass depth occlusion through. This + * keeps its tap-to-place, reticle, coaching and single-shadow-receiver behavior, built from the same public pieces. + */ +@Composable +@Suppress("LongMethod") +private fun DeviceModelArScene(glb: ByteArray, screen: ArScreen, onCapture: (() -> Unit)?) { + val engine = rememberEngine() + val modelLoader = rememberModelLoader(engine) + val materialLoader = rememberMaterialLoader(engine) + val haptic = LocalHapticFeedback.current + var anchor by remember { mutableStateOf(null) } + // Saved across a rotation; the anchor cannot be, since the activity's new ARCore session starts unplaced. + var yawDegrees by rememberSaveable { mutableFloatStateOf(0f) } + var pose by rememberSaveable { mutableStateOf(RestPose.Upright) } + val bounds = remember(glb) { GlbBounds.parse(glb) } + var latestFrame by remember { mutableStateOf(null) } + var session by remember { mutableStateOf(null) } + var viewport by remember { mutableStateOf(IntSize.Zero) } + var reticlePhase by remember { mutableStateOf(ReticlePhase.SEARCHING) } + var cameraReady by remember { mutableStateOf(false) } + var isTracking by remember { mutableStateOf(false) } + var surfaceFound by remember { mutableStateOf(false) } + var trackingFailure by remember { mutableStateOf(null) } + val placed = anchor != null + Box(modifier = Modifier.fillMaxSize()) { + ARSceneView( + modifier = Modifier.fillMaxSize().onSizeChanged { viewport = it }, + engine = engine, + modelLoader = modelLoader, + materialLoader = materialLoader, + instantPlacementMode = Config.InstantPlacementMode.LOCAL_Y_UP, + sessionConfiguration = { arSession, config -> + // Real objects in front of the model hide it; phones without depth just skip it. + if (arSession.isDepthModeSupported(Config.DepthMode.AUTOMATIC)) { + config.depthMode = Config.DepthMode.AUTOMATIC + } + }, + // The grid shows surfaces until the device stands on one, then gets out of the way. + sceneUnderstanding = SceneUnderstanding(occlusion = true, planeVisualization = !placed), + onSessionUpdated = { updated, frame -> + latestFrame = frame + if (session !== updated) session = updated + cameraReady = true + isTracking = frame.camera.trackingState == TrackingState.TRACKING + if (!surfaceFound) { + surfaceFound = + updated.getAllTrackables(Plane::class.java).any { it.trackingState == TrackingState.TRACKING } + } + }, + onTrackingFailureChanged = { trackingFailure = it }, + onGestureListener = + rememberOnGestureListener( + onSingleTapConfirmed = { event, node -> + // A tap on the model is a gesture on it, not a new placement. + val frame = latestFrame + if (node == null && frame != null) { + placementHit(frame, event.x, event.y, instantPlacement = true)?.let { hit -> + anchor?.detach() + anchor = hit.createAnchor() + haptic.performHapticFeedback(HapticFeedbackType.LongPress) + } + } + }, + // A one-finger drag anywhere turns it: a twist on the node itself needs both fingers on a few + // centimeters of device. GestureDetector's distance is previous minus current, so negate it. + onScroll = { _, _, _, distance -> yawDegrees -= distance.x * YAW_DEGREES_PER_PX }, + ), + ) { + if (!placed && isTracking && viewport != IntSize.Zero) { + PlacementReticle( + xPx = viewport.width / 2f, + yPx = viewport.height / 2f, + onHitResultChanged = { hit -> reticlePhase = reticlePhaseFor(hasHit = hit != null) }, + ) { + PlacementReticleVisual(materialLoader = materialLoader, phase = reticlePhase) + } + } + // The plane grid carries its own shadow receiver while it shows; one per plane once it is hidden. + if (placed) { + val planes by rememberDetectedPlanes(session) + planes.forEach { plane -> key(plane) { ShadowReceiverPlane(plane = plane) } } + } + anchor?.let { placedAnchor -> + key(placedAnchor) { + val instance = remember(modelLoader) { modelLoader.createModelInstance(ByteBuffer.wrap(glb)) } + DisposableEffect(instance) { onDispose { modelLoader.destroyModel(instance.model) } } + AnchorNode(anchor = placedAnchor) { + // Moving is a new tap and scaling would break real size, so the drag's yaw and the rest pose + // are the only edits. The yaw wraps the pose so it always turns about the surface's up. + val offset = bounds?.let(pose::offset) ?: floatArrayOf(0f, 0f, 0f) + Node(rotation = Rotation(y = yawDegrees)) { + ModelNode( + modelInstance = instance, + position = Position(offset[0], offset[1], offset[2]), + rotation = Rotation(x = pose.pitchDegrees, z = pose.rollDegrees), + ) { + val glassMin = screen.glassMin + val glassMax = screen.glassMax + val frame = screen.frame + if (glassMin != null && glassMax != null && frame != null) { + DeviceScreenNode(glassMin, glassMax, frame) + } + } + } + } + } + } + } + val guidance = + rememberArGuidanceState( + cameraReady = cameraReady, + isTracking = isTracking, + surfaceFound = surfaceFound, + trackingFailureReason = trackingFailure, + ) + ARCoachingOverlay(guidance) + Column( + modifier = Modifier.align(Alignment.BottomCenter).safeDrawingPadding().padding(bottom = 16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(16.dp), + ) { + if (placed && bounds != null) RestPoseChips(selected = pose, onSelect = { pose = it }) + onCapture?.let { ShutterButton(it) } + } + } +} + +/** Which face of the device lies on the surface. */ +@Composable +private fun RestPoseChips(selected: RestPose, onSelect: (RestPose) -> Unit, modifier: Modifier = Modifier) { + val colors = + FilterChipDefaults.filterChipColors( + labelColor = Color.White, + selectedContainerColor = Color.White.copy(alpha = SCRIM_ALPHA), + selectedLabelColor = Color.White, + ) + Row( + modifier = + modifier + .clip(RoundedCornerShape(24.dp)) + .background(Color.Black.copy(alpha = SCRIM_ALPHA)) + .horizontalScroll(rememberScrollState()) + .padding(horizontal = 8.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + RestPose.entries.forEach { pose -> + FilterChip( + selected = pose == selected, + onClick = { onSelect(pose) }, + label = { Text(stringResource(pose.label)) }, + colors = colors, + ) + } + } +} + +private val RestPose.label: StringResource + get() = + when (this) { + RestPose.Upright -> Res.string.device_model_rest_upright + RestPose.FaceUp -> Res.string.device_model_rest_face_up + RestPose.FaceDown -> Res.string.device_model_rest_face_down + RestPose.LeftSide -> Res.string.device_model_rest_left_side + RestPose.RightSide -> Res.string.device_model_rest_right_side + } + +/** Title bar over the camera; the model's notice opens under it, clear of the controls at the bottom. */ +@Composable +private fun ArHeader(title: String, notice: String?, onDismiss: () -> Unit) { + var noticeShown by rememberSaveable { mutableStateOf(false) } + Column( + modifier = + Modifier.fillMaxWidth().background(Color.Black.copy(alpha = SCRIM_ALPHA)).safeDrawingPadding().padding(4.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + IconButton(onClick = onDismiss) { + Icon(MeshtasticIcons.Close, contentDescription = stringResource(Res.string.close), tint = Color.White) + } + Text( + text = title, + style = MaterialTheme.typography.titleLarge, + color = Color.White, + modifier = Modifier.padding(start = 8.dp).weight(1f), + ) + if (notice != null) { + IconButton(onClick = { noticeShown = !noticeShown }) { + Icon(MeshtasticIcons.Info, contentDescription = stringResource(Res.string.info), tint = Color.White) + } + } + } + AnimatedVisibility(visible = noticeShown && notice != null) { + Text( + text = notice.orEmpty(), + style = MaterialTheme.typography.labelSmall, + color = Color.White, + modifier = Modifier.padding(horizontal = 12.dp, vertical = 4.dp), + ) + } + } +} + +/** A camera-app shutter: the one control you reach for with a thumb, so it sits bottom center. */ +@Composable +private fun ShutterButton(onClick: () -> Unit) { + FilledIconButton( + onClick = onClick, + modifier = Modifier.size(72.dp), + colors = IconButtonDefaults.filledIconButtonColors(containerColor = Color.White, contentColor = Color.Black), + ) { + Icon(MeshtasticIcons.PhotoCamera, contentDescription = stringResource(Res.string.device_model_capture)) + } +} + +/** Saves the AR view to Pictures and says so; Android 10 and later only, where that needs no storage permission. */ +@Composable +private fun rememberArCapture(title: String): (() -> Unit)? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) return null + val view = LocalView.current + val scope = rememberCoroutineScope() + val dispatchers = koinInject() + val haptic = LocalHapticFeedback.current + val saved = stringResource(Res.string.device_model_capture_saved) + val failed = stringResource(Res.string.device_model_capture_failed) + return { + haptic.performHapticFeedback(HapticFeedbackType.LongPress) + scope.launch { + val uri = captureArScene(view.rootView, title, dispatchers.io) + Toast.makeText(view.context, if (uri != null) saved else failed, Toast.LENGTH_SHORT).show() + } + } +} + +/** + * Holds the activity at its current orientation while AR is up. A rotation recreates the activity, and the new ARCore + * session starts without the planes it found or the device placed on one; turning the phone here moves the camera. + */ +@Composable +private fun LockOrientationWhileShown() { + val context = LocalContext.current + DisposableEffect(context) { + val activity = context.findActivity() + val previous = activity?.requestedOrientation + activity?.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_LOCKED + onDispose { if (activity != null && previous != null) activity.requestedOrientation = previous } + } +} + +private tailrec fun Context.findActivity(): Activity? = when (this) { + is Activity -> this + is ContextWrapper -> baseContext.findActivity() + else -> null +} + +/** The display glass in model coordinates and the live frame to draw on it; all null when there is nothing to show. */ +private class ArScreen(val glassMin: FloatArray?, val glassMax: FloatArray?, val frame: MirrorFrame?) diff --git a/androidApp/src/main/AndroidManifest.xml b/androidApp/src/main/AndroidManifest.xml index 11fcb5165a2..97af8afea68 100644 --- a/androidApp/src/main/AndroidManifest.xml +++ b/androidApp/src/main/AndroidManifest.xml @@ -28,6 +28,18 @@ android:name="android.hardware.location.gps" android:required="false" /> + + + + diff --git a/androidApp/src/main/kotlin/org/meshtastic/app/MainActivity.kt b/androidApp/src/main/kotlin/org/meshtastic/app/MainActivity.kt index 827e1125935..b2e77b4f15e 100644 --- a/androidApp/src/main/kotlin/org/meshtastic/app/MainActivity.kt +++ b/androidApp/src/main/kotlin/org/meshtastic/app/MainActivity.kt @@ -57,6 +57,7 @@ import org.koin.compose.viewmodel.koinViewModel import org.meshtastic.app.intro.AnalyticsIntro import org.meshtastic.app.map.getMapViewProvider import org.meshtastic.app.node.component.InlineMap +import org.meshtastic.app.node.deviceModelArViewer import org.meshtastic.app.node.metrics.getTracerouteMapOverlayInsets import org.meshtastic.app.ui.MainScreen import org.meshtastic.core.barcode.rememberBarcodeScanner @@ -86,6 +87,7 @@ import org.meshtastic.core.ui.theme.MODE_DYNAMIC import org.meshtastic.core.ui.util.LocalAnalyticsIntroProvider import org.meshtastic.core.ui.util.LocalBarcodeScannerProvider import org.meshtastic.core.ui.util.LocalBarcodeScannerSupported +import org.meshtastic.core.ui.util.LocalDeviceModelArViewer import org.meshtastic.core.ui.util.LocalDiscoveryMapProvider import org.meshtastic.core.ui.util.LocalEventBranding import org.meshtastic.core.ui.util.LocalInlineMapProvider @@ -268,6 +270,7 @@ class MainActivity : AppCompatActivity() { packageManager.hasSystemFeature(PackageManager.FEATURE_NFC_HOST_CARD_EMULATION), LocalAnalyticsIntroProvider provides { AnalyticsIntro() }, LocalMapViewProvider provides getMapViewProvider(), + LocalDeviceModelArViewer provides deviceModelArViewer, LocalSitePlannerAvailable provides true, LocalInlineMapProvider provides { node, modifier -> InlineMap(node, modifier) }, LocalNodeTrackMapProvider provides diff --git a/androidApp/src/test/kotlin/org/meshtastic/app/SceneViewManifestTest.kt b/androidApp/src/test/kotlin/org/meshtastic/app/SceneViewManifestTest.kt new file mode 100644 index 00000000000..18d24a22f98 --- /dev/null +++ b/androidApp/src/test/kotlin/org/meshtastic/app/SceneViewManifestTest.kt @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.app + +import android.app.Application +import android.content.Context +import android.content.pm.FeatureInfo +import android.content.pm.PackageManager +import androidx.test.core.app.ApplicationProvider +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertNotNull + +/** + * The SceneView AAR's manifest marks GLES 3.0 as a required feature and requests VIBRATE. The 3D viewer is optional, so + * the app manifest neutralizes both; these assertions read the merged manifest and fail if either override is dropped. + */ +@RunWith(RobolectricTestRunner::class) +@Config(application = Application::class, sdk = [34]) +class SceneViewManifestTest { + + private val context: Context = ApplicationProvider.getApplicationContext() + + @Test + fun `gles 3 is declared but not required`() { + val features = + context.packageManager.getPackageInfo(context.packageName, PackageManager.GET_CONFIGURATIONS).reqFeatures + val gles = assertNotNull(features?.firstOrNull { it.reqGlEsVersion == GLES_3_0 }, "no GLES 3.0 uses-feature") + assertFalse( + gles.flags and FeatureInfo.FLAG_REQUIRED != 0, + "GLES 3.0 is required; the viewer must stay optional", + ) + } + + @Test + fun `vibrate is not requested`() { + val permissions = + context.packageManager + .getPackageInfo(context.packageName, PackageManager.GET_PERMISSIONS) + .requestedPermissions + assertFalse(permissions.orEmpty().contains(android.Manifest.permission.VIBRATE)) + } + + private companion object { + const val GLES_3_0 = 0x00030000 + } +} diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt new file mode 100644 index 00000000000..d88940234b3 --- /dev/null +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImpl.kt @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.data.manager + +import co.touchlab.kermit.Logger +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.koin.core.annotation.Single +import org.meshtastic.core.repository.DisplayMirrorManager +import org.meshtastic.core.repository.MirrorFormat +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette +import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette + +/** + * Display frame reassembly state machine. + * + * Chunks of one frame arrive contiguously and in offset order (FromRadio is a reliable ordered stream), so a chunk with + * a new `frame_id` or `offset == 0` starts a new frame and an out-of-sequence chunk drops the partial frame. Calls are + * serialized by the single receive-loop collector in MeshServiceOrchestrator; this singleton's state outlives a + * session, so [reset] must be called when one ends. + */ +@Single +class DisplayMirrorManagerImpl : DisplayMirrorManager { + + private val _frame = MutableStateFlow(null) + override val frame = _frame.asStateFlow() + + private val _palette = MutableStateFlow(null) + override val palette = _palette.asStateFlow() + + private var buffer: ByteArray? = null + private var frameId = 0 + private var received = 0 + private var frameWidth = 0 + private var frameHeight = 0 + + private val rects = MirrorRectCompositor() + + private val paletteRegions = mutableListOf() + private var paletteSignature = 0 + private var paletteReceived = 0 + private var paletteDefaultOn = 0 + private var paletteDefaultOff = 0 + + override fun handleIncomingFrame(chunk: DisplayFrame) { + if (chunk.format == DisplayFrame.Format.RGB565) { + if (rects.handle(chunk)) emitRectFrame(chunk) + } else { + handleMonoChunk(chunk) + } + } + + private fun handleMonoChunk(chunk: DisplayFrame) { + val data = chunk.data_.toByteArray() + val total = chunk.total_size + + if (!isAcceptableChunk(chunk, data.size)) return + + if (chunk.offset == 0) { + buffer = ByteArray(total) + frameId = chunk.frame_id + received = 0 + // Geometry is authoritative on the first chunk; a later chunk may not transpose it. + frameWidth = chunk.width + frameHeight = chunk.height + } + + val buf = buffer + if (buf == null || !matchesCurrentFrame(chunk, buf.size)) { + Logger.w { "DisplayMirror: bad chunk ${chunk.frame_id}@${chunk.offset}, expected $frameId@$received" } + buffer = null + return + } + + data.copyInto(buf, chunk.offset) + received += data.size + + if (received == total) { + _frame.value = + MirrorFrame( + width = frameWidth, + height = frameHeight, + frameId = frameId, + paletteSignature = chunk.palette_signature, + pixels = buf, + ) + buffer = null + } + } + + override fun handleIncomingPalette(chunk: DisplayPalette) { + if (chunk.region_offset == 0 && chunk.region_total <= MAX_PALETTE_REGIONS) { + paletteRegions.clear() + paletteSignature = chunk.signature + paletteReceived = 0 + // Defaults are authoritative on the first chunk; later chunks may omit them. + paletteDefaultOn = chunk.default_on_color + paletteDefaultOff = chunk.default_off_color + } + if (!isAcceptablePaletteChunk(chunk)) { + paletteRegions.clear() + paletteReceived = -1 // poison until the next offset-0 chunk restarts + return + } + paletteRegions.addAll(chunk.regions) + paletteReceived += chunk.regions.size + + if (paletteReceived >= chunk.region_total) { + _palette.value = + MirrorPalette( + signature = chunk.signature, + defaultOnColor = paletteDefaultOn, + defaultOffColor = paletteDefaultOff, + regions = paletteRegions.toList(), + ) + paletteReceived = -1 // complete: a surplus chunk must not extend it; the next palette starts at offset 0 + } + } + + private fun matchesCurrentFrame(chunk: DisplayFrame, bufSize: Int): Boolean { + val inSequence = chunk.frame_id == frameId && chunk.offset == received && bufSize == chunk.total_size + val geometryStable = chunk.width == frameWidth && chunk.height == frameHeight + return inSequence && geometryStable + } + + /** Sequenced continuation of the current palette, within the region cap; logs and rejects anything else. */ + private fun isAcceptablePaletteChunk(chunk: DisplayPalette): Boolean { + val withinCap = + chunk.region_total <= MAX_PALETTE_REGIONS && + paletteReceived >= 0 && + paletteReceived + chunk.regions.size <= MAX_PALETTE_REGIONS + val inSequence = chunk.signature == paletteSignature && chunk.region_offset == paletteReceived + val acceptable = withinCap && inSequence + if (!acceptable) { + Logger.w { "DisplayMirror: bad palette chunk ${chunk.signature}@${chunk.region_offset}" } + } + return acceptable + } + + /** + * Format must be MONO_VLSB and width/height must describe exactly total_size bytes; a zero or lying dimension would + * otherwise reach the renderer (aspectRatio requires > 0). + */ + private fun isAcceptableChunk(chunk: DisplayFrame, dataSize: Int): Boolean { + val total = chunk.total_size + val expectedSize = chunk.width * ((chunk.height + PIXELS_PER_PAGE - 1) / PIXELS_PER_PAGE) + val acceptable = + chunk.format == DisplayFrame.Format.MONO_VLSB && + chunk.width > 0 && + chunk.height > 0 && + expectedSize == total && + total <= MAX_FRAME_BYTES && + chunk.offset + dataSize <= total + if (!acceptable) { + Logger.w { + "DisplayMirror: dropping bad chunk (format=${chunk.format} ${chunk.width}x${chunk.height} " + + "offset=${chunk.offset} size=$dataSize total=$total)" + } + } + return acceptable + } + + private fun emitRectFrame(chunk: DisplayFrame) { + val pixels = rects.snapshot() ?: return + _frame.value = + MirrorFrame( + width = rects.width, + height = rects.height, + frameId = chunk.frame_id, + paletteSignature = 0, + pixels = pixels, + format = MirrorFormat.RGB565, + ) + } + + override fun reset() { + buffer = null + received = 0 + rects.reset() + paletteRegions.clear() + paletteReceived = 0 + paletteSignature = 0 + _frame.value = null + _palette.value = null + } + + private companion object { + // Generous sanity cap: 320x240 at 1bpp is 9600 bytes. + const val MAX_FRAME_BYTES = 16384 + + // The firmware's region table caps at 48; anything past this is a bug or an attack. + const val MAX_PALETTE_REGIONS = 512 + + // MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row). + const val PIXELS_PER_PAGE = 8 + } +} diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt index 0253f8f9063..2ce393781a0 100644 --- a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImpl.kt @@ -22,6 +22,7 @@ import kotlinx.coroutines.SupervisorJob import org.koin.core.annotation.Single import org.meshtastic.core.common.util.ioDispatcher import org.meshtastic.core.model.util.isOtaStatusNotification +import org.meshtastic.core.repository.DisplayMirrorManager import org.meshtastic.core.repository.FirmwareUpdateStatusRepository import org.meshtastic.core.repository.FromRadioPacketHandler import org.meshtastic.core.repository.LockdownCoordinator @@ -44,6 +45,8 @@ import org.meshtastic.core.resources.key_verification_request_title import org.meshtastic.core.resources.key_verification_title import org.meshtastic.core.resources.low_entropy_key_title import org.meshtastic.proto.ClientNotification +import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette import org.meshtastic.proto.FromRadio /** Implementation of [FromRadioPacketHandler] that dispatches [FromRadio] variants to specialized handlers. */ @@ -54,6 +57,7 @@ class FromRadioPacketHandlerImpl( private val configFlowManager: Lazy, private val configHandler: Lazy, private val xmodemManager: Lazy, + private val displayMirrorManager: Lazy, private val mqttManager: MqttManager, private val packetHandler: PacketHandler, private val serviceNotifications: MeshNotificationManager, @@ -83,6 +87,8 @@ class FromRadioPacketHandlerImpl( val regionPresets = proto.region_presets val xmodemPacket = proto.xmodemPacket val lockdownStatus = proto.lockdown_status + val displayFrame = proto.display_frame + val displayPalette = proto.display_palette when { myInfo != null -> configFlowManager.value.handleMyInfo(myInfo, session) @@ -130,6 +136,10 @@ class FromRadioPacketHandlerImpl( xmodemPacket != null -> runIfSessionActive(session, "XModem packet") { xmodemManager.value.handleIncomingXModem(xmodemPacket) } + displayFrame != null -> handleDisplayFrame(displayFrame, session) + + displayPalette != null -> handleDisplayPalette(displayPalette, session) + lockdownStatus != null -> runIfSessionActive(session, "lockdown status") { lockdownCoordinator.handleLockdownStatus(lockdownStatus) @@ -146,6 +156,12 @@ class FromRadioPacketHandlerImpl( } } + private fun handleDisplayFrame(frame: DisplayFrame, session: RadioSessionContext) = + runIfSessionActive(session, "display frame") { displayMirrorManager.value.handleIncomingFrame(frame) } + + private fun handleDisplayPalette(palette: DisplayPalette, session: RadioSessionContext) = + runIfSessionActive(session, "display palette") { displayMirrorManager.value.handleIncomingPalette(palette) } + private fun runIfSessionActive(session: RadioSessionContext, operation: String, block: () -> Unit) { if (!radioInterfaceService.runIfSessionActive(session, block)) { Logger.d { "Discarding $operation from stale transport session" } diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/MirrorRectCompositor.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/MirrorRectCompositor.kt new file mode 100644 index 00000000000..171f0a106bf --- /dev/null +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/manager/MirrorRectCompositor.kt @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.data.manager + +import co.touchlab.kermit.Logger +import org.meshtastic.proto.DisplayFrame + +private data class RectArea( + val x: Int, + val y: Int, + val width: Int, + val height: Int, + val panelWidth: Int, + val panelHeight: Int, +) + +/** + * Reassembles RGB565 dirty-rect chunks (LVGL-based device UIs) and composites them into a persistent little-endian + * RGB565 canvas. Rect geometry is latched from the chunk at offset 0 so a later chunk cannot transpose the blit, and + * every field is validated before anything is allocated — `total_size` is device-controlled. + */ +internal class MirrorRectCompositor { + + var width = 0 + private set + + var height = 0 + private set + + private var canvas: ByteArray? = null + private var buffer: ByteArray? = null + private var received = 0 + private var frameId = 0 + private var area: RectArea? = null + + /** Feeds one chunk; returns true when it completed a rect and the canvas changed. */ + fun handle(chunk: DisplayFrame): Boolean { + val data = chunk.data_.toByteArray() + if (!isAcceptable(chunk, data.size)) { + buffer = null + return false + } + if (chunk.offset == 0) { + buffer = ByteArray(chunk.total_size) + frameId = chunk.frame_id + received = 0 + area = RectArea(chunk.rect_x, chunk.rect_y, rectWidth(chunk), rectHeight(chunk), chunk.width, chunk.height) + } + return accumulate(chunk, data) + } + + private fun accumulate(chunk: DisplayFrame, data: ByteArray): Boolean { + val buf = buffer ?: return false + data.copyInto(buf, chunk.offset) + received += data.size + val complete = received >= buf.size + if (complete) { + composite(buf) + buffer = null + } + return complete + } + + /** Snapshot of the composited canvas, or null before the first completed rect. */ + fun snapshot(): ByteArray? = canvas?.copyOf() + + fun reset() { + canvas = null + buffer = null + received = 0 + frameId = 0 + width = 0 + height = 0 + area = null + } + + /** A rect that omits width or height covers the full panel in that axis. */ + private fun rectWidth(chunk: DisplayFrame) = if (chunk.rect_width > 0) chunk.rect_width else chunk.width + + private fun rectHeight(chunk: DisplayFrame) = if (chunk.rect_height > 0) chunk.rect_height else chunk.height + + private fun isAcceptable(chunk: DisplayFrame, dataSize: Int): Boolean { + val expected = if (withinPanel(chunk)) rectWidth(chunk) * rectHeight(chunk) * 2 else -1 + val sized = expected == chunk.total_size && chunk.total_size in 1..MAX_RECT_BYTES + // Continuations must match the buffer and the panel latched at offset 0, not merely their own claim. + val latched = area + val inSequence = + chunk.offset == 0 || + ( + buffer?.size == chunk.total_size && + chunk.frame_id == frameId && + chunk.offset == received && + latched != null && + chunk.width == latched.panelWidth && + chunk.height == latched.panelHeight + ) + val acceptable = sized && inSequence && chunk.offset + dataSize <= chunk.total_size + if (!acceptable) { + Logger.w { + "DisplayMirror: dropping bad rect ${chunk.rect_width}x${chunk.rect_height} total=${chunk.total_size}" + } + } + return acceptable + } + + /** Subtraction form throughout: `rect_x + rect_width` overflows for hostile values. */ + private fun withinPanel(chunk: DisplayFrame): Boolean { + val w = rectWidth(chunk) + val h = rectHeight(chunk) + return chunk.width in 1..MAX_PANEL_EDGE && + chunk.height in 1..MAX_PANEL_EDGE && + w in 1..chunk.width && + h in 1..chunk.height && + chunk.rect_x in 0..(chunk.width - w) && + chunk.rect_y in 0..(chunk.height - h) + } + + /** Uses the panel latched at offset 0, the geometry the rect was validated against. */ + private fun composite(rect: ByteArray) { + val blit = area ?: return + if (canvas == null || width != blit.panelWidth || height != blit.panelHeight) { + canvas = ByteArray(blit.panelWidth * blit.panelHeight * 2) + width = blit.panelWidth + height = blit.panelHeight + } + val target = canvas ?: return + val rowBytes = blit.width * 2 + for (row in 0 until blit.height) { + val src = row * rowBytes + val dst = ((blit.y + row) * width + blit.x) * 2 + rect.copyInto(target, dst, src, src + rowBytes) + } + } + + private companion object { + // Sanity bound for RGB565 panels (largest realistic is 800x480). + const val MAX_PANEL_EDGE = 1024 + + // A single rect can at most cover the largest panel we accept. + const val MAX_RECT_BYTES = MAX_PANEL_EDGE * MAX_PANEL_EDGE * 2 + } +} diff --git a/core/data/src/commonMain/kotlin/org/meshtastic/core/data/repository/DeviceModelRepositoryImpl.kt b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/repository/DeviceModelRepositoryImpl.kt new file mode 100644 index 00000000000..f735bb00cee --- /dev/null +++ b/core/data/src/commonMain/kotlin/org/meshtastic/core/data/repository/DeviceModelRepositoryImpl.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.data.repository + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.koin.core.annotation.Single +import org.meshtastic.core.network.DeviceModelRemoteDataSource +import org.meshtastic.core.repository.DeviceModelRepository + +/** + * Memoizes per image name for the process lifetime: a model is a few hundred KB to a few MB and the hero re-asks on + * every node detail open. A failed fetch is not cached, so a later open retries. + */ +@Single +class DeviceModelRepositoryImpl(private val remote: DeviceModelRemoteDataSource) : DeviceModelRepository { + private val lock = Mutex() + private val existing = mutableSetOf() + private val models = mutableMapOf() + + override suspend fun hasModel(imageFile: String): Boolean = lock.withLock { imageFile in existing } || + remote.exists(imageFile).also { exists -> if (exists) lock.withLock { existing += imageFile } } + + override suspend fun getModel(imageFile: String): ByteArray? = lock.withLock { models[imageFile] } + ?: remote.fetch(imageFile)?.also { bytes -> + lock.withLock { + models[imageFile] = bytes + existing += imageFile + } + } +} diff --git a/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt new file mode 100644 index 00000000000..5a59165eeb7 --- /dev/null +++ b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/DisplayMirrorManagerImplTest.kt @@ -0,0 +1,379 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.data.manager + +import okio.ByteString.Companion.toByteString +import org.meshtastic.core.repository.MirrorFormat +import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class DisplayMirrorManagerImplTest { + + private val manager = DisplayMirrorManagerImpl() + + // 128x64 MONO_VLSB frame = 1024 bytes; the firmware chunks at 384. + private fun chunk( + frameId: Int = 1, + offset: Int = 0, + data: ByteArray, + total: Int = FRAME_BYTES, + width: Int = WIDTH, + height: Int = HEIGHT, + format: DisplayFrame.Format = DisplayFrame.Format.MONO_VLSB, + ) = DisplayFrame.Builder() + .also { wb -> + wb.width = width + wb.height = height + wb.format = format + wb.frame_id = frameId + wb.offset = offset + wb.total_size = total + wb.data_ = data.toByteString() + } + .build() + + private fun bytes(size: Int, fill: Int) = ByteArray(size) { fill.toByte() } + + private fun palette( + signature: Int, + regionOffset: Int, + regionTotal: Int, + regions: List, + ) = DisplayPalette.Builder() + .also { wb -> + wb.signature = signature + wb.region_offset = regionOffset + wb.region_total = regionTotal + wb.regions = regions + } + .build() + + @Test + fun `reassembles a three-chunk frame in order`() { + manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1))) + manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2))) + assertNull(manager.frame.value) + + manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3))) + + val frame = manager.frame.value!! + assertEquals(WIDTH, frame.width) + assertEquals(HEIGHT, frame.height) + assertEquals(1, frame.frameId) + assertContentEquals(bytes(384, 1) + bytes(384, 2) + bytes(256, 3), frame.pixels) + } + + @Test + fun `single-chunk frame completes immediately`() { + // 64x32 = 256 bytes fits one chunk + manager.handleIncomingFrame(chunk(width = 64, height = 32, total = 256, data = bytes(256, 7))) + + assertEquals(256, manager.frame.value?.pixels?.size) + } + + @Test + fun `out-of-sequence chunk drops the partial frame`() { + manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1))) + manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3))) // gap: 384 missing + + manager.handleIncomingFrame(chunk(offset = 384, data = bytes(384, 2))) // too late + assertNull(manager.frame.value) + } + + @Test + fun `offset zero restarts mid-frame on device reboot or torn capture`() { + manager.handleIncomingFrame(chunk(frameId = 5, offset = 0, data = bytes(384, 1))) + + // New frame id starting over at offset 0 wins + manager.handleIncomingFrame(chunk(frameId = 6, offset = 0, data = bytes(384, 4))) + manager.handleIncomingFrame(chunk(frameId = 6, offset = 384, data = bytes(384, 5))) + manager.handleIncomingFrame(chunk(frameId = 6, offset = 768, data = bytes(256, 6))) + + assertEquals(6, manager.frame.value?.frameId) + } + + @Test + fun `frame id change mid-frame without offset zero is dropped`() { + manager.handleIncomingFrame(chunk(frameId = 1, offset = 0, data = bytes(384, 1))) + manager.handleIncomingFrame(chunk(frameId = 2, offset = 384, data = bytes(384, 2))) + + assertNull(manager.frame.value) + } + + @Test + fun `rejects geometry that does not match total size`() { + manager.handleIncomingFrame( + chunk(width = 64, height = 64, total = 256, data = bytes(256, 1)), + ) // 64x64 needs 512 + manager.handleIncomingFrame(chunk(width = 0, height = 64, total = 0, data = ByteArray(0))) + manager.handleIncomingFrame(chunk(width = 128, height = 0, total = 0, data = ByteArray(0))) + + assertNull(manager.frame.value) + } + + @Test + fun `rejects unsupported format`() { + manager.handleIncomingFrame( + chunk( + width = 64, + height = 32, + total = 256, + data = bytes(256, 1), + format = DisplayFrame.Format.FORMAT_UNSPECIFIED, + ), + ) + + assertNull(manager.frame.value) + } + + @Test + fun `reassembles a two-chunk palette and tags frames with its signature`() { + val region = + DisplayPalette.ColorRegion.Builder() + .also { wb -> + wb.x = 0 + wb.y = 0 + wb.width = 128 + wb.height = 16 + wb.on_color = 0xF800 + wb.off_color = 0 + } + .build() + manager.handleIncomingPalette( + DisplayPalette.Builder() + .also { wb -> + wb.signature = 42 + wb.default_on_color = 0xFFFF + wb.default_off_color = 0 + wb.region_offset = 0 + wb.region_total = 3 + wb.regions = listOf(region, region) + } + .build(), + ) + assertNull(manager.palette.value) + + manager.handleIncomingPalette( + palette(signature = 42, regionOffset = 2, regionTotal = 3, regions = listOf(region)), + ) + + val palette = manager.palette.value!! + assertEquals(42, palette.signature) + assertEquals(0xFFFF, palette.defaultOnColor) + assertEquals(3, palette.regions.size) + } + + @Test + fun `out-of-sequence palette chunk is dropped until a restart`() { + val region = + DisplayPalette.ColorRegion.Builder() + .also { wb -> + wb.x = 0 + wb.y = 0 + wb.width = 8 + wb.height = 8 + } + .build() + manager.handleIncomingPalette( + palette(signature = 7, regionOffset = 0, regionTotal = 2, regions = listOf(region)), + ) + manager.handleIncomingPalette( + palette(signature = 7, regionOffset = 5, regionTotal = 2, regions = listOf(region)), + ) + assertNull(manager.palette.value) + + // A fresh offset-0 chunk recovers + manager.handleIncomingPalette( + palette(signature = 8, regionOffset = 0, regionTotal = 1, regions = listOf(region)), + ) + assertEquals(8, manager.palette.value?.signature) + } + + @Test + fun `reset clears frames palettes and partial state`() { + manager.handleIncomingFrame(chunk(width = 64, height = 32, total = 256, data = bytes(256, 7))) + manager.handleIncomingPalette(palette(signature = 9, regionOffset = 0, regionTotal = 0, regions = emptyList())) + + manager.reset() + + assertNull(manager.frame.value) + assertNull(manager.palette.value) + } + + @Test + fun `rejects palettes past the region cap`() { + val region = + DisplayPalette.ColorRegion.Builder() + .also { wb -> + wb.x = 0 + wb.y = 0 + wb.width = 8 + wb.height = 8 + } + .build() + manager.handleIncomingPalette( + palette(signature = 3, regionOffset = 0, regionTotal = 100_000, regions = listOf(region)), + ) + + assertNull(manager.palette.value) + } + + @Test + fun `chunk that changes geometry mid-frame is dropped`() { + manager.handleIncomingFrame(chunk(offset = 0, data = bytes(384, 1))) + // Transposed geometry with the same total size must not complete the frame + manager.handleIncomingFrame(chunk(width = 64, height = 128, offset = 384, data = bytes(384, 2))) + manager.handleIncomingFrame(chunk(offset = 768, data = bytes(256, 3))) + + assertNull(manager.frame.value) + } + + // ── RGB565 dirty-rect path ────────────────────────────────────────────── + + private fun rectChunk( + frameId: Int = 1, + offset: Int = 0, + data: ByteArray, + total: Int = 200, + width: Int = WIDTH, + height: Int = HEIGHT, + rectX: Int = 0, + rectY: Int = 0, + rectW: Int = 10, + rectH: Int = 10, + ) = DisplayFrame.Builder() + .also { wb -> + wb.width = width + wb.height = height + wb.format = DisplayFrame.Format.RGB565 + wb.frame_id = frameId + wb.offset = offset + wb.total_size = total + wb.rect_x = rectX + wb.rect_y = rectY + wb.rect_width = rectW + wb.rect_height = rectH + wb.data_ = data.toByteString() + } + .build() + + @Test + fun `composites a completed rect into an RGB565 frame`() { + manager.handleIncomingFrame(rectChunk(data = bytes(200, 0x11))) + + val frame = manager.frame.value!! + assertEquals(MirrorFormat.RGB565, frame.format) + assertEquals(WIDTH * HEIGHT * 2, frame.pixels.size) + } + + @Test + fun `rejects a rect whose total size does not match its geometry`() { + manager.handleIncomingFrame(rectChunk(total = 999, data = bytes(200, 1))) + + assertNull(manager.frame.value) + } + + @Test + fun `rejects a rect that would overflow the panel`() { + manager.handleIncomingFrame(rectChunk(rectX = WIDTH - 2, data = bytes(200, 1))) + manager.handleIncomingFrame(rectChunk(rectY = Int.MAX_VALUE, data = bytes(200, 1))) + + assertNull(manager.frame.value) + } + + @Test + fun `rejects an oversized rect without allocating`() { + manager.handleIncomingFrame( + rectChunk( + width = 4000, + height = 4000, + rectW = 4000, + rectH = 4000, + total = Int.MAX_VALUE, + data = bytes(8, 1), + ), + ) + + assertNull(manager.frame.value) + } + + @Test + fun `continuation chunk cannot grow the rect buffer`() { + manager.handleIncomingFrame(rectChunk(offset = 0, total = 200, data = bytes(100, 1))) + // Same geometry claim, larger total: must not write past the allocation. + manager.handleIncomingFrame(rectChunk(offset = 100, total = 800, rectW = 20, rectH = 20, data = bytes(100, 2))) + + assertNull(manager.frame.value) + } + + @Test + fun `continuation chunk cannot change the panel the rect was checked against`() { + manager.handleIncomingFrame(rectChunk(offset = 0, total = 200, data = bytes(100, 1))) + // A smaller panel on the final chunk would blit the 10x10 rect past the end of the canvas. + manager.handleIncomingFrame(rectChunk(offset = 100, total = 200, width = 10, height = 10, data = bytes(100, 2))) + + assertNull(manager.frame.value) + } + + @Test + fun `a surplus chunk after a completed palette does not extend it`() { + val region = + DisplayPalette.ColorRegion.Builder() + .also { wb -> + wb.width = 8 + wb.height = 8 + } + .build() + manager.handleIncomingPalette( + palette(signature = 5, regionOffset = 0, regionTotal = 1, regions = listOf(region)), + ) + manager.handleIncomingPalette( + palette(signature = 5, regionOffset = 1, regionTotal = 1, regions = listOf(region)), + ) + + assertEquals(1, manager.palette.value?.regions?.size) + } + + @Test + fun `an over-cap restart leaves the published palette intact`() { + val region = + DisplayPalette.ColorRegion.Builder() + .also { wb -> + wb.width = 8 + wb.height = 8 + } + .build() + manager.handleIncomingPalette( + palette(signature = 5, regionOffset = 0, regionTotal = 1, regions = listOf(region)), + ) + manager.handleIncomingPalette( + palette(signature = 6, regionOffset = 0, regionTotal = 100_000, regions = listOf(region)), + ) + + assertEquals(5, manager.palette.value?.signature) + } + + private companion object { + const val WIDTH = 128 + const val HEIGHT = 64 + const val FRAME_BYTES = 1024 + } +} diff --git a/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImplTest.kt b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImplTest.kt index 18644aade4c..8ae93b41e0a 100644 --- a/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImplTest.kt +++ b/core/data/src/commonTest/kotlin/org/meshtastic/core/data/manager/FromRadioPacketHandlerImplTest.kt @@ -27,6 +27,7 @@ import dev.mokkery.verify import dev.mokkery.verify.VerifyMode import dev.mokkery.verifySuspend import org.meshtastic.core.model.util.isOtaStatusNotification +import org.meshtastic.core.repository.DisplayMirrorManager import org.meshtastic.core.repository.FirmwareUpdateStatusRepository import org.meshtastic.core.repository.MeshConfigFlowManager import org.meshtastic.core.repository.MeshConfigHandler @@ -43,6 +44,8 @@ import org.meshtastic.proto.Channel import org.meshtastic.proto.ClientNotification import org.meshtastic.proto.Config import org.meshtastic.proto.DeviceMetadata +import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette import org.meshtastic.proto.FromRadio import org.meshtastic.proto.LoRaRegionPresetMap import org.meshtastic.proto.LockdownStatus @@ -68,6 +71,7 @@ class FromRadioPacketHandlerImplTest { private val configFlowManager: MeshConfigFlowManager = mock(MockMode.autofill) private val configHandler: MeshConfigHandler = mock(MockMode.autofill) private val xmodemManager: XModemManager = mock(MockMode.autofill) + private val displayMirrorManager: DisplayMirrorManager = mock(MockMode.autofill) private val lockdownCoordinator = FakeLockdownCoordinator() private val firmwareUpdateStatusRepository = FirmwareUpdateStatusRepository() private val radioInterfaceService: RadioInterfaceService = mock(MockMode.autofill) @@ -105,6 +109,7 @@ class FromRadioPacketHandlerImplTest { lazy { configFlowManager }, lazy { configHandler }, lazy { xmodemManager }, + lazy { displayMirrorManager }, mqttManager, packetHandler, serviceNotifications, @@ -216,17 +221,62 @@ class FromRadioPacketHandlerImplTest { val xmodemPacket = XModem.Builder().build() val lockdownStatus = LockdownStatus.Builder().also { wb -> wb.state = LockdownStatus.State.LOCKED }.build() + val displayFrame = + DisplayFrame.Builder() + .also { wb -> + wb.width = 128 + wb.height = 64 + } + .build() + val displayPalette = DisplayPalette.Builder().also { wb -> wb.signature = 1 }.build() + handle(FromRadio.Builder().also { wb -> wb.mqttClientProxyMessage = proxyMessage }.build()) handle(FromRadio.Builder().also { wb -> wb.queueStatus = queueStatus }.build()) handle(FromRadio.Builder().also { wb -> wb.xmodemPacket = xmodemPacket }.build()) handle(FromRadio.Builder().also { wb -> wb.lockdown_status = lockdownStatus }.build()) + handle(FromRadio.Builder().also { wb -> wb.display_frame = displayFrame }.build()) + handle(FromRadio.Builder().also { wb -> wb.display_palette = displayPalette }.build()) verify(mode = VerifyMode.exactly(0)) { mqttManager.handleMqttProxyMessage(any()) } verify(mode = VerifyMode.exactly(0)) { packetHandler.handleQueueStatus(any()) } verify(mode = VerifyMode.exactly(0)) { xmodemManager.handleIncomingXModem(any()) } + verify(mode = VerifyMode.exactly(0)) { displayMirrorManager.handleIncomingFrame(any()) } + verify(mode = VerifyMode.exactly(0)) { displayMirrorManager.handleIncomingPalette(any()) } assertEquals(null, lockdownCoordinator.lastStatus) } + @Test + fun `handleFromRadio routes DISPLAY_FRAME to displayMirrorManager`() { + val frame = + DisplayFrame.Builder() + .also { wb -> + wb.width = 128 + wb.height = 64 + wb.frame_id = 1 + wb.total_size = 1024 + } + .build() + + handle(FromRadio.Builder().also { wb -> wb.display_frame = frame }.build()) + + verify { displayMirrorManager.handleIncomingFrame(frame) } + } + + @Test + fun `handleFromRadio routes DISPLAY_PALETTE to displayMirrorManager`() { + val palette = + DisplayPalette.Builder() + .also { wb -> + wb.signature = 7 + wb.region_total = 0 + } + .build() + + handle(FromRadio.Builder().also { wb -> wb.display_palette = palette }.build()) + + verify { displayMirrorManager.handleIncomingPalette(palette) } + } + @Test fun `handleFromRadio routes LOCKDOWN_STATUS to lockdownCoordinator`() { val lockdownStatus = diff --git a/core/model/src/commonMain/kotlin/org/meshtastic/core/model/DeviceHardware.kt b/core/model/src/commonMain/kotlin/org/meshtastic/core/model/DeviceHardware.kt index a3877e277c4..94d7dd79e3b 100644 --- a/core/model/src/commonMain/kotlin/org/meshtastic/core/model/DeviceHardware.kt +++ b/core/model/src/commonMain/kotlin/org/meshtastic/core/model/DeviceHardware.kt @@ -70,3 +70,10 @@ data class DeviceHardware( val isDiy: Boolean get() = tags.orEmpty().contains("DIY") } + +/** + * The illustration the apps show for this hardware: the cased variant when the registry lists one (index 1), else the + * bare board. The flasher's 3D model, when there is one, is named after the same file. + */ +val DeviceHardware.heroImageFile: String? + get() = images?.getOrNull(1) ?: images?.getOrNull(0) diff --git a/core/network/src/commonMain/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSource.kt b/core/network/src/commonMain/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSource.kt new file mode 100644 index 00000000000..d673e563261 --- /dev/null +++ b/core/network/src/commonMain/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSource.kt @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.network + +import io.ktor.client.HttpClient +import io.ktor.client.request.get +import io.ktor.client.request.head +import io.ktor.client.statement.HttpResponse +import io.ktor.client.statement.readRawBytes +import io.ktor.http.ContentType +import io.ktor.http.contentType +import io.ktor.http.isSuccess +import kotlinx.coroutines.withContext +import org.koin.core.annotation.Single +import org.meshtastic.core.common.util.safeCatching +import org.meshtastic.core.di.CoroutineDispatchers + +/** Fetches the flasher's GLB device models; the URL is the illustration's with the extension swapped. */ +@Single +class DeviceModelRemoteDataSource(private val client: HttpClient, private val dispatchers: CoroutineDispatchers) { + + /** + * True when the model is served. The flasher is a single-page app that answers a missing path with 200 and its HTML + * shell, so only the GLB content type counts. + */ + suspend fun exists(imageFile: String): Boolean = withContext(dispatchers.io) { + safeCatching { client.head(modelUrl(imageFile)).isGlb() }.getOrDefault(false) + } + + /** The model bytes, or null when absent or the fetch failed. */ + suspend fun fetch(imageFile: String): ByteArray? = withContext(dispatchers.io) { + safeCatching { + val response = client.get(modelUrl(imageFile)) + if (response.isGlb()) response.readRawBytes() else null + } + .getOrNull() + } + + private fun HttpResponse.isGlb(): Boolean = + status.isSuccess() && contentType()?.withoutParameters() == GLB_CONTENT_TYPE + + companion object { + private val GLB_CONTENT_TYPE = ContentType("model", "gltf-binary") + + fun modelUrl(imageFile: String): String = + "${HttpClientDefaults.DEVICE_MODEL_BASE_URL}/${imageFile.substringBeforeLast('.')}.glb" + } +} diff --git a/core/network/src/commonMain/kotlin/org/meshtastic/core/network/HttpClientDefaults.kt b/core/network/src/commonMain/kotlin/org/meshtastic/core/network/HttpClientDefaults.kt index 6e1e2a0bb90..38790a5f4e4 100644 --- a/core/network/src/commonMain/kotlin/org/meshtastic/core/network/HttpClientDefaults.kt +++ b/core/network/src/commonMain/kotlin/org/meshtastic/core/network/HttpClientDefaults.kt @@ -66,6 +66,9 @@ object HttpClientDefaults { * those fall back to the release zip named by the API's `zip_url`, which is a GitHub release asset and unaffected. */ const val RELEASE_BASE_URL = "https://release.meshtastic.org" + + /** The flasher's device models, `.glb` beside the `img/devices/.svg` illustrations the apps show. */ + const val DEVICE_MODEL_BASE_URL = "https://flasher.meshtastic.org/models/devices" } /** diff --git a/core/network/src/commonTest/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSourceTest.kt b/core/network/src/commonTest/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSourceTest.kt new file mode 100644 index 00000000000..b15cafd5725 --- /dev/null +++ b/core/network/src/commonTest/kotlin/org/meshtastic/core/network/DeviceModelRemoteDataSourceTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.network + +import io.ktor.client.HttpClient +import io.ktor.client.engine.mock.MockEngine +import io.ktor.client.engine.mock.respond +import io.ktor.http.HttpHeaders +import io.ktor.http.headersOf +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.test.runTest +import org.meshtastic.core.di.CoroutineDispatchers +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class DeviceModelRemoteDataSourceTest { + private val dispatchers = + CoroutineDispatchers(io = Dispatchers.Default, main = Dispatchers.Default, default = Dispatchers.Default) + + @Test + fun modelUrlSwapsTheIllustrationExtension() { + assertEquals( + "https://flasher.meshtastic.org/models/devices/rak4631_case.glb", + DeviceModelRemoteDataSource.modelUrl("rak4631_case.svg"), + ) + } + + @Test + fun aServedGlbExistsAndIsFetched() = runTest { + val glb = byteArrayOf(0x67, 0x6C, 0x54, 0x46) + withSource(respondWith = glb, contentType = "model/gltf-binary") { source -> + assertTrue(source.exists("tracker-t1000-e.svg")) + assertContentEquals(glb, source.fetch("tracker-t1000-e.svg")) + } + } + + @Test + fun theFlashersHtmlShellForAMissingModelIsNotAModel() = runTest { + withSource( + respondWith = "".encodeToByteArray(), + contentType = "text/html;charset=utf-8", + ) { source -> + assertFalse(source.exists("t-deck.svg")) + assertNull(source.fetch("t-deck.svg")) + } + } + + private suspend fun withSource( + respondWith: ByteArray, + contentType: String, + block: suspend (DeviceModelRemoteDataSource) -> Unit, + ) { + val client = + HttpClient(MockEngine { respond(respondWith, headers = headersOf(HttpHeaders.ContentType, contentType)) }) + try { + block(DeviceModelRemoteDataSource(client, dispatchers)) + } finally { + client.close() + } + } +} diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt index 9e4adf7fee5..f94c5b717c5 100644 --- a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/AdminController.kt @@ -51,6 +51,23 @@ interface AdminController { /** Updates a local radio channel. Same fire-and-forget contract as [setLocalConfig]. */ suspend fun setLocalChannel(channel: Channel) + /** + * Enables or disables live display mirroring on the locally connected node. While enabled, the device streams its + * framebuffer as `FromRadio.display_frame` chunks — collect them via [DisplayMirrorManager]. Non-suspend (immediate + * send, bypassing the outbound FIFO) so lifecycle teardown can reliably disable the stream. + */ + fun setDisplayMirror(enabled: Boolean) + + /** Requests a single framebuffer frame from the locally connected node, delivered like mirrored frames. */ + fun requestDisplayFrame() + + /** + * Injects a physical input event (button/key/touch) into the locally connected node's UI via the firmware + * InputBroker. Event codes are the firmware's `input_broker_event` values. Non-suspend immediate send: input must + * not queue behind bulk admin traffic. + */ + fun sendInputEvent(eventCode: Int, kbChar: Int = 0, touchX: Int = 0, touchY: Int = 0) + // ── Remote configuration ──────────────────────────────────────────────── /** Updates the owner (user info) on a remote node. */ diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DeviceModelRepository.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DeviceModelRepository.kt new file mode 100644 index 00000000000..219f293b57e --- /dev/null +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DeviceModelRepository.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.repository + +/** + * The flasher's real-size 3D device models (`/models/devices/.glb`), one per device illustration. A hardware + * entry's image file names its model by extension swap, and only some illustrations have one. + */ +interface DeviceModelRepository { + /** Whether a model exists for [imageFile] (an entry of `DeviceHardware.images`). False on any network failure. */ + suspend fun hasModel(imageFile: String): Boolean + + /** The GLB bytes for [imageFile], or null when there is no model or it could not be fetched. */ + suspend fun getModel(imageFile: String): ByteArray? +} diff --git a/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt new file mode 100644 index 00000000000..ba9bb59bb10 --- /dev/null +++ b/core/repository/src/commonMain/kotlin/org/meshtastic/core/repository/DisplayMirrorManager.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.repository + +import kotlinx.coroutines.flow.StateFlow +import org.meshtastic.proto.DisplayFrame +import org.meshtastic.proto.DisplayPalette + +/** + * Reassembles the chunked [DisplayFrame] stream from the connected device into complete framebuffer snapshots. + * + * The device streams its 1bpp framebuffer as `FromRadio.display_frame` chunks while display mirroring is enabled (see + * `AdminMessage.set_display_mirror`). Chunks of one frame share a `frame_id`; a frame is complete when `offset + + * data.size == total_size`. + */ +interface DisplayMirrorManager { + /** Latest completely reassembled frame, or null before the first one arrives. */ + val frame: StateFlow + + /** Latest completely reassembled color palette, or null when the device renders monochrome. */ + val palette: StateFlow + + /** Routes an incoming display frame chunk from the device to the reassembly state machine. */ + fun handleIncomingFrame(chunk: DisplayFrame) + + /** Routes an incoming palette chunk from the device to the reassembly state machine. */ + fun handleIncomingPalette(chunk: DisplayPalette) + + /** + * Clears all held state — latest frame, latest palette, and partial reassembly buffers. Call when a session ends or + * a fresh mirror UI opens, so a previous device's final screen is never shown as if live. + */ + fun reset() +} + +/** + * A complete color palette for [MirrorFrame]s whose `paletteSignature` matches [signature]: per-region RGB565 on/off + * colors (later regions override earlier ones where they overlap) plus defaults for pixels outside all regions. + */ +data class MirrorPalette( + val signature: Int, + val defaultOnColor: Int, + val defaultOffColor: Int, + val regions: List, +) + +/** Pixel encodings a [MirrorFrame] can carry. */ +enum class MirrorFormat { + /** 1 bit per pixel, vertical LSB-first pages — byte index `x + (y / 8) * width`, bit index `y % 8`. */ + MONO_VLSB, + + /** 16 bits per pixel, little-endian RGB565, rows tightly packed. */ + RGB565, +} + +/** One complete device framebuffer snapshot in [format]. */ +data class MirrorFrame( + val width: Int, + val height: Int, + val frameId: Int, + val paletteSignature: Int, + val pixels: ByteArray, + val format: MirrorFormat = MirrorFormat.MONO_VLSB, +) { + override fun equals(other: Any?): Boolean = other is MirrorFrame && + other.width == width && + other.height == height && + other.frameId == frameId && + other.paletteSignature == paletteSignature && + other.format == format && + other.pixels.contentEquals(pixels) + + override fun hashCode(): Int { + var result = width + result = 31 * result + height + result = 31 * result + frameId + result = 31 * result + paletteSignature + result = 31 * result + format.ordinal + result = 31 * result + pixels.contentHashCode() + return result + } +} diff --git a/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_left.xml b/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_left.xml new file mode 100644 index 00000000000..2d023ab1074 --- /dev/null +++ b/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_left.xml @@ -0,0 +1,9 @@ + + + diff --git a/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_right.xml b/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_right.xml new file mode 100644 index 00000000000..30d777e4324 --- /dev/null +++ b/core/resources/src/commonMain/composeResources/drawable/ic_keyboard_arrow_right.xml @@ -0,0 +1,9 @@ + + + diff --git a/core/resources/src/commonMain/composeResources/drawable/ic_photo_camera.xml b/core/resources/src/commonMain/composeResources/drawable/ic_photo_camera.xml new file mode 100644 index 00000000000..dcbda4fc875 --- /dev/null +++ b/core/resources/src/commonMain/composeResources/drawable/ic_photo_camera.xml @@ -0,0 +1,12 @@ + + + + + + diff --git a/core/resources/src/commonMain/composeResources/drawable/ic_view_in_ar.xml b/core/resources/src/commonMain/composeResources/drawable/ic_view_in_ar.xml new file mode 100644 index 00000000000..d2d37ed3317 --- /dev/null +++ b/core/resources/src/commonMain/composeResources/drawable/ic_view_in_ar.xml @@ -0,0 +1,10 @@ + + + + diff --git a/core/resources/src/commonMain/composeResources/files/environments/studio_small_09.hdr b/core/resources/src/commonMain/composeResources/files/environments/studio_small_09.hdr new file mode 100644 index 00000000000..e9a2d2272ce Binary files /dev/null and b/core/resources/src/commonMain/composeResources/files/environments/studio_small_09.hdr differ diff --git a/core/resources/src/commonMain/composeResources/values/strings.xml b/core/resources/src/commonMain/composeResources/values/strings.xml index e0cb031919b..1d69df69d4b 100644 --- a/core/resources/src/commonMain/composeResources/values/strings.xml +++ b/core/resources/src/commonMain/composeResources/values/strings.xml @@ -371,6 +371,7 @@ Disable to skip writing mesh logs to disk Store mesh logs App logs + Mirror Packets A link is asking this app to connect to the device at %1$s. Only continue if you trust where the link came from. Connect to this device? @@ -424,6 +425,17 @@ %1$s %1$s: %2$s%% %1$s: %2$s V + Take a picture + Couldn't save the picture + Saved to Pictures + Couldn't load the 3D model + Face down + Face up + Left side + Right side + Upright + View 3D model + View in your space Device sleeping Device Storage & UI (Read-Only) Theme: %1$s, Language: %2$s @@ -1219,6 +1231,24 @@ µg/m³ Min Minimum wake time + + Mirroring + Back + Device screen + E-ink display: refresh is slow — prefer one-shot Refresh over continuous mirroring. + %1$d×%2$d · frame #%3$d + Arrows · Enter = OK · Esc = back + Click the screen for keyboard control, or swipe it to navigate + Tap the screen to touch the device, or swipe to navigate + Down + Left + OK + Right + Up + Keyboard + No frame received yet — enable mirroring or tap Refresh. + Not connected to a device. + Mirror off Presets Module configuration Modules already unlocked diff --git a/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt b/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt index d47c3fdfb79..883d0d76ce7 100644 --- a/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt +++ b/core/service/src/commonMain/kotlin/org/meshtastic/core/service/AdminControllerImpl.kt @@ -139,6 +139,33 @@ internal class AdminControllerImpl( scope.handledLaunch { radioConfigRepository.setLocalConfig(config) } } + override fun setDisplayMirror(enabled: Boolean) { + commandSender.sendAdminImmediate(myNodeNum) { + AdminMessage.Builder().also { wb -> wb.set_display_mirror = enabled }.build() + } + } + + override fun requestDisplayFrame() { + commandSender.sendAdminImmediate(myNodeNum) { + AdminMessage.Builder().also { wb -> wb.get_display_frame_request = true }.build() + } + } + + override fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) { + val inputEvent = + AdminMessage.InputEvent.Builder() + .also { wb -> + wb.event_code = eventCode + wb.kb_char = kbChar + wb.touch_x = touchX + wb.touch_y = touchY + } + .build() + commandSender.sendAdminImmediate(myNodeNum) { + AdminMessage.Builder().also { wb -> wb.send_input_event = inputEvent }.build() + } + } + override suspend fun setConfig(destNum: Int, config: Config, packetId: Int) { commandSender.sendAdmin(destNum, packetId) { AdminMessage.Builder().also { wb -> wb.set_config = config }.build() diff --git a/core/testing/src/commonMain/kotlin/org/meshtastic/core/testing/FakeRadioController.kt b/core/testing/src/commonMain/kotlin/org/meshtastic/core/testing/FakeRadioController.kt index 691d88e05db..0089f6b3333 100644 --- a/core/testing/src/commonMain/kotlin/org/meshtastic/core/testing/FakeRadioController.kt +++ b/core/testing/src/commonMain/kotlin/org/meshtastic/core/testing/FakeRadioController.kt @@ -276,6 +276,12 @@ class FakeRadioController : override suspend fun refreshMetadata(destNum: Int) {} + override fun setDisplayMirror(enabled: Boolean) {} + + override fun requestDisplayFrame() {} + + override fun sendInputEvent(eventCode: Int, kbChar: Int, touchX: Int, touchY: Int) {} + override suspend fun setLocalConfig(config: Config) { if (throwOnSetLocalConfig) error("Fake local config write failure") if (rejectLocalConfigWritesRemaining > 0) { diff --git a/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Hardware.kt b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Hardware.kt index fb9634e63d9..2449adfb9f0 100644 --- a/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Hardware.kt +++ b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Hardware.kt @@ -29,16 +29,26 @@ import org.meshtastic.core.resources.ic_display_settings import org.meshtastic.core.resources.ic_memory import org.meshtastic.core.resources.ic_mesh_radio import org.meshtastic.core.resources.ic_nfc +import org.meshtastic.core.resources.ic_photo_camera import org.meshtastic.core.resources.ic_settings_input_antenna import org.meshtastic.core.resources.ic_speaker_phone import org.meshtastic.core.resources.ic_terminal import org.meshtastic.core.resources.ic_usb import org.meshtastic.core.resources.ic_usb_off +import org.meshtastic.core.resources.ic_view_in_ar import org.meshtastic.core.resources.ic_wifi /** The maker hardware mark from meshtastic/design#160: a wrench, Material Symbols "build". */ val MeshtasticIcons.Wrench: ImageVector @Composable get() = vectorResource(Res.drawable.ic_build) + +/** Material Symbols "view_in_ar": the entry to a device's 3D model. */ +val MeshtasticIcons.ViewInAr: ImageVector + @Composable get() = vectorResource(Res.drawable.ic_view_in_ar) + +/** Material Symbols "photo_camera": capture the AR view. */ +val MeshtasticIcons.PhotoCamera: ImageVector + @Composable get() = vectorResource(Res.drawable.ic_photo_camera) val MeshtasticIcons.BluetoothConnected: ImageVector @Composable get() = vectorResource(Res.drawable.ic_bluetooth_connected) val MeshtasticIcons.BluetoothSearching: ImageVector diff --git a/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Navigation.kt b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Navigation.kt index 0fa667bfa3d..afbb17d7594 100644 --- a/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Navigation.kt +++ b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/icon/Navigation.kt @@ -26,6 +26,8 @@ import org.meshtastic.core.resources.ic_chevron_right import org.meshtastic.core.resources.ic_expand_less import org.meshtastic.core.resources.ic_expand_more import org.meshtastic.core.resources.ic_keyboard_arrow_down +import org.meshtastic.core.resources.ic_keyboard_arrow_left +import org.meshtastic.core.resources.ic_keyboard_arrow_right import org.meshtastic.core.resources.ic_keyboard_arrow_up val MeshtasticIcons.ArrowBack: ImageVector @@ -36,6 +38,10 @@ val MeshtasticIcons.KeyboardArrowDown: ImageVector @Composable get() = vectorResource(Res.drawable.ic_keyboard_arrow_down) val MeshtasticIcons.KeyboardArrowUp: ImageVector @Composable get() = vectorResource(Res.drawable.ic_keyboard_arrow_up) +val MeshtasticIcons.KeyboardArrowLeft: ImageVector + @Composable get() = vectorResource(Res.drawable.ic_keyboard_arrow_left) +val MeshtasticIcons.KeyboardArrowRight: ImageVector + @Composable get() = vectorResource(Res.drawable.ic_keyboard_arrow_right) val MeshtasticIcons.ArrowDownward: ImageVector @Composable get() = vectorResource(Res.drawable.ic_arrow_downward) val MeshtasticIcons.ExpandMore: ImageVector diff --git a/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/util/LocalDeviceModelArViewer.kt b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/util/LocalDeviceModelArViewer.kt new file mode 100644 index 00000000000..5bac0cb13c3 --- /dev/null +++ b/core/ui/src/commonMain/kotlin/org/meshtastic/core/ui/util/LocalDeviceModelArViewer.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.core.ui.util + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.staticCompositionLocalOf +import org.meshtastic.core.repository.MirrorFrame + +/** Null where there is no AR runtime at all: desktop and the fdroid flavor. */ +@Suppress("CompositionLocalAllowlist") // A host-provided capability, like the barcode and NFC providers. +val LocalDeviceModelArViewer = staticCompositionLocalOf { null } + +/** Places a device's real-size GLB model in the camera view. */ +interface DeviceModelArViewer { + /** False on a phone the AR runtime has ruled out. Asked when a model opens, never at app start. */ + @Composable fun isAvailable(): Boolean + + /** + * [notice] is the model's own copyright line, shown with it. With [screenMin]/[screenMax] (the display glass, in + * model coordinates) and a [screenFrame], the live device screen is drawn on the model. + */ + @Composable + fun Content( + glb: ByteArray, + title: String, + notice: String?, + onDismiss: () -> Unit, + screenMin: FloatArray?, + screenMax: FloatArray?, + screenFrame: MirrorFrame?, + ) +} diff --git a/feature/node/build.gradle.kts b/feature/node/build.gradle.kts index 050b1b94dfa..c9f4612d3f1 100644 --- a/feature/node/build.gradle.kts +++ b/feature/node/build.gradle.kts @@ -36,10 +36,18 @@ kotlin { implementation(projects.core.ui) implementation(projects.core.di) + implementation(libs.kotlinx.serialization.json) implementation(libs.markdown.renderer) implementation(libs.markdown.renderer.m3) implementation(libs.vico.compose) implementation(libs.vico.compose.m3) + // The desktop renderer's POM drags a mac-only Compose runtime onto every OS's classpath. The + // string form is the one overload that takes an exclude; the version still comes from the catalog. + implementation("io.github.sceneview:sceneview-compose:${libs.versions.sceneview.get()}") { + exclude(group = "org.jetbrains.compose.desktop", module = "desktop-jvm-macos-arm64") + // Fuel 2.3.1 (the Android renderer's URL loader, unused here) ships the pre-Parcelize runtime. + exclude(group = "org.jetbrains.kotlin", module = "kotlin-android-extensions-runtime") + } // JetBrains Material 3 Adaptive (multiplatform ListDetailPaneScaffold) implementation(libs.jetbrains.compose.material3.adaptive) @@ -48,7 +56,13 @@ kotlin { implementation(libs.jetbrains.compose.material3.adaptive.navigation3) } - androidMain.dependencies { implementation(libs.markdown.renderer.android) } + androidMain.dependencies { + implementation(libs.markdown.renderer.android) + // The Android renderer under sceneview-compose, which only ships it at runtime scope. + implementation("io.github.sceneview:sceneview:${libs.versions.sceneview.get()}") { + exclude(group = "org.jetbrains.kotlin", module = "kotlin-android-extensions-runtime") + } + } // Compose UI tests live in jvmTest, not commonTest: this module enables android host tests, and the // androidHostTest stubs leave Build.FINGERPRINT null, which the Compose Robolectric idling strategy NPEs on. diff --git a/feature/node/src/androidMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.android.kt b/feature/node/src/androidMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.android.kt new file mode 100644 index 00000000000..a9a34e751ae --- /dev/null +++ b/feature/node/src/androidMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.android.kt @@ -0,0 +1,192 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import android.graphics.Bitmap +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.SideEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.produceState +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import com.google.android.filament.Engine +import io.github.sceneview.SceneScope +import io.github.sceneview.SceneView +import io.github.sceneview.createDefaultCameraManipulator +import io.github.sceneview.environment.Environment +import io.github.sceneview.gesture.CameraGestureDetector +import io.github.sceneview.loaders.EnvironmentLoader +import io.github.sceneview.loaders.ModelLoader +import io.github.sceneview.math.Direction +import io.github.sceneview.math.Position +import io.github.sceneview.math.Size +import io.github.sceneview.model.model +import io.github.sceneview.rememberEngine +import io.github.sceneview.rememberEnvironmentLoader +import io.github.sceneview.rememberModelLoader +import org.meshtastic.core.common.util.safeCatching +import org.meshtastic.core.repository.MirrorFormat +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.feature.node.model.GlbBounds +import java.nio.ByteBuffer +import io.github.sceneview.node.ImageNode as ImageNodeImpl + +/** + * SceneView directly rather than the multiplatform SceneViewer, whose Android HDR path swaps out and frees its + * placeholder environment while the scene still renders it, a native crash. The probe loads before the scene exists, so + * nothing is swapped. SceneView centers the model on the origin, so the orbit targets the origin. + */ +@Composable +internal actual fun DeviceModelScene( + glb: ByteArray, + bounds: GlbBounds, + onError: () -> Unit, + modifier: Modifier, + visible: Boolean, + screen: GlbBounds?, + screenFrame: MirrorFrame?, +) { + val engine = rememberEngine() + val modelLoader = rememberModelLoader(engine) + val environmentLoader = rememberEnvironmentLoader(engine) + val currentOnError by rememberUpdatedState(onError) + val environment by + produceState(null, environmentLoader) { + value = + safeCatching { environmentLoader.loadHDREnvironment(STUDIO_LIGHT_PROBE, createSkybox = false) } + .getOrNull() + .also { if (it == null) currentOnError() } + } + // Captured, not read through the delegate: the null effect's dispose runs after the state already holds the load. + val loadedEnvironment = environment + DisposableEffect(loadedEnvironment) { onDispose { loadedEnvironment?.let(environmentLoader::destroyEnvironment) } } + val cameraManipulator = + remember(bounds) { + createDefaultCameraManipulator(eyePosition = orbitEye(bounds), targetPosition = Position(0f)) + } + // Hidden, only the scene goes: the engine, its compiled materials, the probe and the camera stay for when it shows + // again. + if (visible && loadedEnvironment != null) { + OrbitScene( + glb, + engine, + modelLoader, + environmentLoader, + loadedEnvironment, + cameraManipulator, + onError, + modifier, + ) { + if (screen != null && screenFrame != null) DeviceScreenNode(screen.min, screen.max, screenFrame) + } + } +} + +/** + * The model instance does not survive its scene's teardown, so it is rebuilt from the bytes each time, which is cheap. + */ +@Composable +@Suppress("LongParameterList") +private fun OrbitScene( + glb: ByteArray, + engine: Engine, + modelLoader: ModelLoader, + environmentLoader: EnvironmentLoader, + environment: Environment, + cameraManipulator: CameraGestureDetector.CameraManipulator, + onError: () -> Unit, + modifier: Modifier = Modifier, + overlay: @Composable SceneScope.() -> Unit = {}, +) { + val currentOnError by rememberUpdatedState(onError) + val instance = + remember(modelLoader, glb) { + runCatching { modelLoader.createModelInstance(ByteBuffer.wrap(glb)) }.getOrNull() + } + DisposableEffect(instance) { onDispose { instance?.let { modelLoader.destroyModel(it.model) } } } + SideEffect { if (instance == null) currentOnError() } + if (instance == null) return + SceneView( + modifier = modifier, + engine = engine, + modelLoader = modelLoader, + environmentLoader = environmentLoader, + environment = environment, + isOpaque = false, + cameraManipulator = cameraManipulator, + ) { + ModelNode(modelInstance = instance) + overlay() + } +} + +/** + * The frame on a plane just proud of the display glass, facing +Z like the flasher's models. The node is created once + * and fed new bitmaps: replacing it would make SceneView re-center the whole scene on every frame. + */ +@Composable +fun SceneScope.DeviceScreenNode(screenMin: FloatArray, screenMax: FloatArray, frame: MirrorFrame) { + val screen = GlbBounds(screenMin, screenMax) + val bitmap = remember(frame) { frame.toBitmap() } ?: return + var node by remember { mutableStateOf(null) } + val first = remember { bitmap } + ImageNode( + bitmap = first, + size = Size(screen.max[0] - screen.min[0], screen.max[1] - screen.min[1]), + center = Position(screen.center[0], screen.center[1], screen.max[2] + SCREEN_LIFT_METERS), + normal = Direction(0f, 0f, 1f), + apply = { node = this }, + ) + // A swapped texture is invisible to SceneView's on-demand loop until something asks for a frame. + SideEffect { + node + ?.takeIf { it.bitmap !== bitmap } + ?.let { + it.bitmap = bitmap + it.requestRender() + } + } +} + +private const val SCREEN_LIFT_METERS = 0.0001f +private const val PIXELS_PER_PAGE = 8 +private const val PIXEL_ON = 0xFFE8F4FF.toInt() +private const val PIXEL_OFF = 0xFF05070A.toInt() + +/** MONO_VLSB only: byte `(y / 8) * width + x`, bit `y % 8`. */ +private fun MirrorFrame.toBitmap(): Bitmap? { + if (format != MirrorFormat.MONO_VLSB || pixels.size < (height + PIXELS_PER_PAGE - 1) / PIXELS_PER_PAGE * width) { + return null + } + val argb = + IntArray(width * height) { i -> + val x = i % width + val y = i / width + val set = pixels[(y / PIXELS_PER_PAGE) * width + x].toInt() and (1 shl (y % PIXELS_PER_PAGE)) != 0 + if (set) PIXEL_ON else PIXEL_OFF + } + return Bitmap.createBitmap(argb, width, height, Bitmap.Config.ARGB_8888) +} + +private fun orbitEye(bounds: GlbBounds): Position { + val eye = orbitEyeOffset(bounds) + return Position(eye[0], eye[1], eye[2]) +} diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceDetailsSection.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceDetailsSection.kt index 7f54afca207..0b385f552ab 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceDetailsSection.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceDetailsSection.kt @@ -17,6 +17,7 @@ package org.meshtastic.feature.node.component import androidx.compose.foundation.background +import androidx.compose.foundation.clickable import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column @@ -39,6 +40,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.semantics.Role import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import coil3.compose.AsyncImage @@ -47,8 +49,10 @@ import coil3.request.ImageRequest import org.jetbrains.compose.resources.stringResource import org.meshtastic.core.model.DeviceHardware import org.meshtastic.core.model.HardwareSupportTier +import org.meshtastic.core.model.heroImageFile import org.meshtastic.core.model.supportTier import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.device_model_view import org.meshtastic.core.resources.ic_unverified import org.meshtastic.core.resources.img_hw_unknown import org.meshtastic.core.resources.supported @@ -56,6 +60,7 @@ import org.meshtastic.core.resources.supported_by_community import org.meshtastic.core.resources.supported_by_maker import org.meshtastic.core.ui.icon.MeshtasticIcons import org.meshtastic.core.ui.icon.Verified +import org.meshtastic.core.ui.icon.ViewInAr import org.meshtastic.core.ui.icon.Wrench import org.meshtastic.core.ui.theme.StatusColors.StatusGreen import org.meshtastic.core.ui.theme.StatusColors.StatusRed @@ -71,12 +76,13 @@ internal fun DeviceHeroSection( deviceHardware: DeviceHardware, reportedTarget: String?, modifier: Modifier = Modifier, + onOpenDeviceModel: (() -> Unit)? = null, ) { Row( modifier = modifier.fillMaxWidth().padding(horizontal = 20.dp, vertical = 12.dp), verticalAlignment = Alignment.CenterVertically, ) { - DeviceAvatar(bgColor, deviceHardware, size = 80) + DeviceAvatar(bgColor, deviceHardware, size = 80, onOpenDeviceModel = onOpenDeviceModel) Spacer(Modifier.width(16.dp)) @@ -94,16 +100,44 @@ internal fun DeviceHeroSection( } } +/** The illustration, and when a 3D model exists a tap target for it with a corner badge saying so. */ @Composable -private fun DeviceAvatar(bgColor: Long, deviceHardware: DeviceHardware, size: Int = 64) { - Box( - modifier = - Modifier.size(size.dp) - .clip(CircleShape) - .background(color = Color(bgColor).copy(alpha = .5f), shape = CircleShape), - contentAlignment = Alignment.Center, - ) { - DeviceHardwareImage(deviceHardware, Modifier.fillMaxSize()) +private fun DeviceAvatar( + bgColor: Long, + deviceHardware: DeviceHardware, + size: Int = 64, + onOpenDeviceModel: (() -> Unit)? = null, +) { + val label = stringResource(Res.string.device_model_view) + Box(modifier = Modifier.size(size.dp)) { + Box( + modifier = + Modifier.fillMaxSize() + .clip(CircleShape) + .background(color = Color(bgColor).copy(alpha = .5f), shape = CircleShape) + .then( + if (onOpenDeviceModel != null) { + Modifier.clickable(onClickLabel = label, role = Role.Button, onClick = onOpenDeviceModel) + } else { + Modifier + }, + ), + contentAlignment = Alignment.Center, + ) { + DeviceHardwareImage(deviceHardware, Modifier.fillMaxSize()) + } + if (onOpenDeviceModel != null) { + Icon( + imageVector = MeshtasticIcons.ViewInAr, + contentDescription = null, + tint = colorScheme.onPrimaryContainer, + modifier = + Modifier.align(Alignment.BottomEnd) + .size(24.dp) + .background(colorScheme.primaryContainer, CircleShape) + .padding(4.dp), + ) + } } } @@ -149,7 +183,7 @@ private fun SupportStatusBadge(tier: HardwareSupportTier) { @Composable internal fun DeviceHardwareImage(deviceHardware: DeviceHardware, modifier: Modifier = Modifier) { - val hwImg = deviceHardware.images?.getOrNull(1) ?: deviceHardware.images?.getOrNull(0) ?: "unknown.svg" + val hwImg = deviceHardware.heroImageFile ?: "unknown.svg" val imageUrl = "https://flasher.meshtastic.org/img/devices/$hwImg" val fallbackPainter = org.jetbrains.compose.resources.painterResource(Res.drawable.img_hw_unknown) AsyncImage( diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelDialog.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelDialog.kt new file mode 100644 index 00000000000..5d481047e53 --- /dev/null +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelDialog.kt @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.safeDrawingPadding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.Icon +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.SideEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import androidx.compose.ui.window.Dialog +import androidx.compose.ui.window.DialogProperties +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import org.jetbrains.compose.resources.stringResource +import org.koin.compose.viewmodel.koinViewModel +import org.meshtastic.core.model.DeviceHardware +import org.meshtastic.core.model.heroImageFile +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.close +import org.meshtastic.core.resources.device_model_load_failed +import org.meshtastic.core.resources.device_model_view_in_space +import org.meshtastic.core.resources.info +import org.meshtastic.core.ui.icon.Close +import org.meshtastic.core.ui.icon.Info +import org.meshtastic.core.ui.icon.MeshtasticIcons +import org.meshtastic.core.ui.icon.ViewInAr +import org.meshtastic.core.ui.util.LocalDeviceModelArViewer + +/** The hardware's real-size 3D model, orbitable, over the whole window; the illustration when it cannot be shown. */ +@Composable +fun DeviceModelDialog( + deviceHardware: DeviceHardware, + onDismiss: () -> Unit, + modifier: Modifier = Modifier, + mirrorScreen: Boolean = false, +) { + val viewModel = koinViewModel() + val state by viewModel.state.collectAsStateWithLifecycle() + val screenFrame = + rememberMirroredScreen(viewModel, mirrorScreen && (state as? DeviceModelState.Ready)?.screen != null) + val imageFile = modelImageFile(deviceHardware) + // Both calls are idempotent, so running on every composition is safe. + SideEffect { if (imageFile != null) viewModel.load(imageFile) else viewModel.markFailed() } + + val arViewer = LocalDeviceModelArViewer.current?.takeIf { it.isAvailable() } + var inAr by rememberSaveable { mutableStateOf(false) } + val arModel = (state as? DeviceModelState.Ready)?.takeIf { inAr && arViewer != null } + + Dialog(onDismissRequest = onDismiss, properties = DialogProperties(usePlatformDefaultWidth = false)) { + // The orbit view stays composed under AR with its surface dropped, so coming back does not reload the model. + Box(modifier = modifier.fillMaxSize()) { + OrbitContent( + deviceHardware = deviceHardware, + state = state, + screenFrame = screenFrame, + onError = viewModel::markFailed, + onDismiss = onDismiss, + onOpenAr = if (arViewer != null) ({ inAr = true }) else null, + visible = arModel == null, + ) + arModel?.let { + arViewer?.Content( + glb = it.glb, + title = deviceHardware.displayName, + notice = it.copyright, + onDismiss = { inAr = false }, + screenMin = it.screen?.min, + screenMax = it.screen?.max, + screenFrame = screenFrame, + ) + } + } + } +} + +@Composable +private fun OrbitContent( + deviceHardware: DeviceHardware, + state: DeviceModelState, + screenFrame: MirrorFrame?, + onError: () -> Unit, + onDismiss: () -> Unit, + onOpenAr: (() -> Unit)?, + visible: Boolean, +) { + val ready = state as? DeviceModelState.Ready + Surface(modifier = Modifier.fillMaxSize(), color = MaterialTheme.colorScheme.background) { + Column(modifier = Modifier.fillMaxSize().safeDrawingPadding()) { + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 4.dp, vertical = 4.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + IconButton(onClick = onDismiss) { + Icon(MeshtasticIcons.Close, contentDescription = stringResource(Res.string.close)) + } + Text( + text = deviceHardware.displayName, + style = MaterialTheme.typography.titleLarge, + modifier = Modifier.padding(start = 8.dp).weight(1f), + ) + if (onOpenAr != null && ready != null) { + IconButton(onClick = onOpenAr) { + Icon( + MeshtasticIcons.ViewInAr, + contentDescription = stringResource(Res.string.device_model_view_in_space), + ) + } + } + } + Box(modifier = Modifier.fillMaxSize(), contentAlignment = Alignment.Center) { + when (val current = state) { + DeviceModelState.Loading -> CircularProgressIndicator() + + DeviceModelState.Failed -> ModelFallback(deviceHardware) + + is DeviceModelState.Ready -> + Box(modifier = Modifier.fillMaxSize()) { + DeviceModelScene( + glb = current.glb, + bounds = current.bounds, + onError = onError, + modifier = Modifier.fillMaxSize(), + visible = visible, + screen = current.screen, + screenFrame = screenFrame, + ) + current.copyright?.let { + ModelNotice(it, Modifier.align(Alignment.BottomEnd).padding(16.dp)) + } + } + } + } + } + } +} + +/** Arms the connected device's screen mirror while [enabled]; null when it is off. */ +@Composable +private fun rememberMirroredScreen(viewModel: DeviceModelViewModel, enabled: Boolean): MirrorFrame? { + val frame by viewModel.screenFrame.collectAsStateWithLifecycle() + DisposableEffect(enabled) { + viewModel.setMirror(enabled) + onDispose { viewModel.setMirror(false) } + } + return frame.takeIf { enabled } +} + +// SPIKE: the hardware list gives the Pocket the bare RAK4631 board's image; the flasher has its own model. +private fun modelImageFile(deviceHardware: DeviceHardware): String? = + if (deviceHardware.platformioTarget == "rak_wismesh_pocket") { + "rak_wismesh_pocket.svg" + } else { + deviceHardware.heroImageFile + } + +/** + * The model file's own copyright notice behind an info button, shut until tapped, as the maps carry their credit. The + * text is the owner's, so it is never translated or cut. + */ +@Composable +fun ModelNotice( + notice: String, + modifier: Modifier = Modifier, + contentColor: Color = MaterialTheme.colorScheme.onSurfaceVariant, + containerColor: Color = MaterialTheme.colorScheme.surfaceContainerHigh, +) { + var expanded by rememberSaveable { mutableStateOf(false) } + // The toggle is last, so the notice opens inboard of it from the trailing corner, in RTL too. + Row( + modifier = modifier.clip(RoundedCornerShape(24.dp)).background(containerColor), + verticalAlignment = Alignment.CenterVertically, + ) { + AnimatedVisibility(visible = expanded, modifier = Modifier.weight(1f, fill = false)) { + Text( + text = notice, + style = MaterialTheme.typography.labelSmall, + color = contentColor, + modifier = Modifier.padding(start = 16.dp, top = 8.dp, bottom = 8.dp), + ) + } + IconButton(onClick = { expanded = !expanded }) { + Icon(MeshtasticIcons.Info, contentDescription = stringResource(Res.string.info), tint = contentColor) + } + } +} + +@Composable +private fun ModelFallback(deviceHardware: DeviceHardware) { + Column(horizontalAlignment = Alignment.CenterHorizontally) { + DeviceHardwareImage(deviceHardware, Modifier.size(240.dp)) + Spacer(Modifier.height(16.dp)) + Text( + text = stringResource(Res.string.device_model_load_failed), + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelViewModel.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelViewModel.kt new file mode 100644 index 00000000000..1ea1b67ecc5 --- /dev/null +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/DeviceModelViewModel.kt @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import org.koin.core.annotation.KoinViewModel +import org.meshtastic.core.repository.AdminController +import org.meshtastic.core.repository.DeviceModelRepository +import org.meshtastic.core.repository.DisplayMirrorManager +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.RadioConfigRepository +import org.meshtastic.feature.node.model.GlbBounds +import kotlin.time.Duration.Companion.seconds + +sealed interface DeviceModelState { + data object Loading : DeviceModelState + + data object Failed : DeviceModelState + + /** [copyright] is the model's own notice, shown wherever the model is; [screen] is its display glass, if marked. */ + data class Ready( + val imageFile: String, + val glb: ByteArray, + val bounds: GlbBounds, + val copyright: String?, + val screen: GlbBounds? = null, + ) : DeviceModelState { + override fun equals(other: Any?): Boolean = + this === other || (other is Ready && imageFile == other.imageFile && bounds == other.bounds) + + override fun hashCode(): Int = 31 * imageFile.hashCode() + bounds.hashCode() + } +} + +/** Fetches a device's GLB and the bounds the viewer frames it with; one model at a time, the last request wins. */ +@KoinViewModel +class DeviceModelViewModel( + private val repository: DeviceModelRepository, + private val displayMirrorManager: DisplayMirrorManager, + private val adminController: AdminController, + radioConfigRepository: RadioConfigRepository, +) : ViewModel() { + private val _state = MutableStateFlow(DeviceModelState.Loading) + val state: StateFlow = _state.asStateFlow() + + /** The connected device's live screen while [setMirror] has it armed. */ + val screenFrame: StateFlow = displayMirrorManager.frame + + // Arming is an admin write, which managed mode forbids. + private val managed: StateFlow = + radioConfigRepository.localConfigFlow + .map { it.security?.is_managed == true } + .stateIn(viewModelScope, SharingStarted.Eagerly, false) + + private var mirroring = false + private var pendingDisarm: Job? = null + private var loading: Job? = null + private var loadedImageFile: String? = null + + fun load(imageFile: String) { + if (loadedImageFile == imageFile) return + loadedImageFile = imageFile + loading?.cancel() + _state.value = DeviceModelState.Loading + loading = viewModelScope.launch { + val glb = repository.getModel(imageFile) + val bounds = glb?.let(GlbBounds::parse) + _state.value = + if (glb != null && bounds != null) { + DeviceModelState.Ready(imageFile, glb, bounds, GlbBounds.copyright(glb), GlbBounds.screen(glb)) + } else { + DeviceModelState.Failed + } + } + } + + /** The renderer rejected a model the parser accepted: show the illustration instead. */ + fun markFailed() { + _state.value = DeviceModelState.Failed + } + + fun setMirror(enabled: Boolean) { + if (enabled) { + pendingDisarm?.cancel() + pendingDisarm = null + arm() + } else if (mirroring && pendingDisarm == null) { + // A rotation tears the dialog down and builds it again at once; keep the stream over that gap rather than + // disarming, re-arming and showing a blank screen until the device repaints. + pendingDisarm = viewModelScope.launch { + delay(MIRROR_DISARM_GRACE) + disarm() + } + } + } + + private fun arm() { + if (mirroring || managed.value) return + displayMirrorManager.reset() + adminController.setDisplayMirror(true) + mirroring = true + } + + private fun disarm() { + pendingDisarm = null + if (!mirroring) return + adminController.setDisplayMirror(false) + mirroring = false + } + + override fun onCleared() { + pendingDisarm?.cancel() + disarm() + } +} + +private val MIRROR_DISARM_GRACE = 3.seconds diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/NodeDetailsSection.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/NodeDetailsSection.kt index 1edb22398a1..fc678a0a8e5 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/NodeDetailsSection.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/NodeDetailsSection.kt @@ -121,6 +121,8 @@ fun NodeDetailsSection( reportedTarget: String? = null, isLocal: Boolean = false, onRequestUserInfo: (() -> Unit)? = null, + hasDeviceModel: Boolean = false, + onOpenDeviceModel: ((DeviceHardware) -> Unit)? = null, ) { SectionCard(title = Res.string.details, modifier = modifier) { Column(modifier = Modifier.animateContentSize()) { @@ -133,6 +135,8 @@ fun NodeDetailsSection( bgColor = node.colors.second.toLong(), deviceHardware = deviceHardware, reportedTarget = reportedTarget, + onOpenDeviceModel = + onOpenDeviceModel?.takeIf { hasDeviceModel }?.let { open -> { open(deviceHardware) } }, ) SectionDivider() } diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.kt new file mode 100644 index 00000000000..87eba3dec75 --- /dev/null +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import dev.romainguy.kotlin.math.Float3 +import io.github.sceneview.compose.EnvironmentSource +import io.github.sceneview.compose.Lighting +import io.github.sceneview.compose.ModelSource +import io.github.sceneview.compose.SceneViewer +import io.github.sceneview.compose.rememberCameraState +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.feature.node.model.GlbBounds +import kotlin.math.PI +import kotlin.math.cos +import kotlin.math.sin + +/** A CC0 studio light probe (Poly Haven, Studio Small 09), at the path both renderers resolve a bundled file by. */ +internal const val STUDIO_LIGHT_PROBE = + "composeResources/org.meshtastic.core.resources/files/environments/studio_small_09.hdr" + +private const val CAMERA_DISTANCE_PER_EXTENT = 2.2f +private const val CAMERA_AZIMUTH_DEGREES = 30f +private const val CAMERA_ELEVATION_DEGREES = 20f +private const val SUN_LUX = 40_000f +private const val DEGREES_TO_RADIANS = (PI / 180.0).toFloat() + +/** The camera's offset from the model's center: the same three-quarter view on every platform. */ +internal fun orbitEyeOffset(bounds: GlbBounds): FloatArray { + val distance = bounds.extent * CAMERA_DISTANCE_PER_EXTENT + val azimuth = CAMERA_AZIMUTH_DEGREES * DEGREES_TO_RADIANS + val elevation = CAMERA_ELEVATION_DEGREES * DEGREES_TO_RADIANS + val horizontal = distance * cos(elevation) + return floatArrayOf(horizontal * sin(azimuth), distance * sin(elevation), horizontal * cos(azimuth)) +} + +/** + * The orbitable model, lit by the studio probe over a transparent backdrop so the theme shows behind it. When both + * [screen] and [screenFrame] are present, the frame is drawn on the model's display glass. + */ +@Composable +internal expect fun DeviceModelScene( + glb: ByteArray, + bounds: GlbBounds, + onError: () -> Unit, + modifier: Modifier = Modifier, + visible: Boolean = true, + screen: GlbBounds? = null, + screenFrame: MirrorFrame? = null, +) + +/** + * The multiplatform SceneViewer, which places the model at its authored coordinates on desktop: the orbit target is the + * bounds center and the distance a multiple of the longest side. + */ +@Composable +internal fun SceneViewerModelScene( + glb: ByteArray, + bounds: GlbBounds, + onError: () -> Unit, + modifier: Modifier = Modifier, +) { + val center = bounds.center + val camera = + rememberCameraState( + target = Float3(center[0], center[1], center[2]), + distance = bounds.extent * CAMERA_DISTANCE_PER_EXTENT, + azimuth = CAMERA_AZIMUTH_DEGREES, + elevation = CAMERA_ELEVATION_DEGREES, + ) + SceneViewer( + model = ModelSource.Bytes(glb), + modifier = modifier, + camera = camera, + lighting = Lighting(intensity = SUN_LUX), + environment = EnvironmentSource.Hdr(STUDIO_LIGHT_PROBE, showSkybox = false), + onError = { onError() }, + ) +} diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailContent.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailContent.kt index 0bb67c73df8..fd0f9d897df 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailContent.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailContent.kt @@ -110,6 +110,8 @@ fun NodeDetailList( onRequestUserInfo = { onAction(NodeDetailAction.HandleNodeMenuAction(NodeMenuAction.RequestUserInfo(node))) }, + hasDeviceModel = uiState.metricsState.hasDeviceModel, + onOpenDeviceModel = { onAction(NodeDetailAction.OpenDeviceModel(it)) }, ) } if (uiState.metricsState.deviceLinks.isNotEmpty()) { diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailScreens.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailScreens.kt index c5ba3964370..366dfa63937 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailScreens.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/detail/NodeDetailScreens.kt @@ -30,6 +30,7 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.runtime.saveable.rememberSaveable import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.unit.dp @@ -47,6 +48,7 @@ import org.meshtastic.core.ui.util.ActiveWhileStarted import org.meshtastic.feature.node.compass.CompassUiState import org.meshtastic.feature.node.compass.CompassViewModel import org.meshtastic.feature.node.component.CompassSheetContent +import org.meshtastic.feature.node.component.DeviceModelDialog import org.meshtastic.feature.node.component.FirmwareReleaseSheetContent import org.meshtastic.feature.node.component.NodeMenuAction import org.meshtastic.feature.node.model.NodeDetailAction @@ -96,6 +98,8 @@ private fun NodeDetailScaffold( compassViewModel: CompassViewModel? = null, ) { var activeOverlay by remember { mutableStateOf(null) } + // Saved, unlike the overlays: a rotation must not close the model, which the user may be turning the phone for. + var deviceModelOpen by rememberSaveable { mutableStateOf(false) } val actualCompassViewModel = compassViewModel val compassUiState by actualCompassViewModel?.uiState?.collectAsStateWithLifecycle() ?: remember { mutableStateOf(CompassUiState()) } @@ -128,6 +132,8 @@ private fun NodeDetailScaffold( is NodeDetailAction.OpenCompass -> activeOverlay = NodeDetailOverlay.Compass(action.node.num, action.displayUnits) + is NodeDetailAction.OpenDeviceModel -> deviceModelOpen = true + else -> handleNodeAction( action = action, @@ -156,6 +162,9 @@ private fun NodeDetailScaffold( ) { viewModel.handleNodeMenuAction(NodeMenuAction.RequestPosition(it)) } + uiState.metricsState.deviceHardware + ?.takeIf { deviceModelOpen } + ?.let { DeviceModelDialog(it, onDismiss = { deviceModelOpen = false }, mirrorScreen = isLocalNode) } } @OptIn(ExperimentalMaterial3Api::class) diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/domain/usecase/CommonGetNodeDetailsUseCase.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/domain/usecase/CommonGetNodeDetailsUseCase.kt index 31a1e526cf7..a961f5b1514 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/domain/usecase/CommonGetNodeDetailsUseCase.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/domain/usecase/CommonGetNodeDetailsUseCase.kt @@ -34,10 +34,12 @@ import org.meshtastic.core.model.FirmwareRelease import org.meshtastic.core.model.MeshLog import org.meshtastic.core.model.MyNodeInfo import org.meshtastic.core.model.Node +import org.meshtastic.core.model.heroImageFile import org.meshtastic.core.model.util.hasValidEnvironmentMetrics import org.meshtastic.core.model.util.isDirectSignal import org.meshtastic.core.repository.DeviceHardwareRepository import org.meshtastic.core.repository.DeviceLinkRepository +import org.meshtastic.core.repository.DeviceModelRepository import org.meshtastic.core.repository.FirmwareReleaseRepository import org.meshtastic.core.repository.MeshLogRepository import org.meshtastic.core.repository.NodeRepository @@ -67,6 +69,7 @@ constructor( private val radioConfigRepository: RadioConfigRepository, private val deviceHardwareRepository: DeviceHardwareRepository, private val deviceLinkRepository: DeviceLinkRepository, + private val deviceModelRepository: DeviceModelRepository, private val firmwareReleaseRepository: FirmwareReleaseRepository, private val nodeRequestActions: NodeRequestActions, private val localeUnitsProvider: LocaleUnitsProvider, @@ -123,8 +126,8 @@ constructor( IdentityGroup(ourNode, myInfo, profile) } - // 3. Device Hardware (+ msh.to links) — non-blocking Flow derived from stable (hwModel, pioEnv) key. - val hardwareAndLinksFlow: Flow>> = + // 3. Device Hardware (+ msh.to links, 3D model) — non-blocking Flow derived from stable (hwModel, pioEnv) key. + val hardwareAndLinksFlow: Flow = combine(nodeFlow, identityFlow) { node, identity -> val isLocal = node.num == identity.ourNode?.num val pioEnv = if (isLocal) identity.myInfo?.pioEnv else null @@ -138,7 +141,8 @@ constructor( hw?.platformioTarget ?.takeIf { it.isNotBlank() } ?.let { deviceLinkRepository.getLinksForTarget(it) } ?: emptyList() - hw to links + val hasModel = hw?.heroImageFile?.let { deviceModelRepository.hasModel(it) } ?: false + HardwareGroup(hw, links, hasModel) } // 4. Metadata & Request Timestamps @@ -194,9 +198,7 @@ constructor( @Suppress("UNCHECKED_CAST") val requests = args[REQUESTS_INDEX] as Pair, List> - @Suppress("UNCHECKED_CAST") - val hardwareAndLinks = args[HARDWARE_INDEX] as Pair> - val (hw, deviceLinks) = hardwareAndLinks + val (hw, deviceLinks, hasDeviceModel) = args[HARDWARE_INDEX] as HardwareGroup val (trReqs, niReqs) = requests val isLocal = node.num == identity.ourNode?.num @@ -212,6 +214,7 @@ constructor( isLocal = isLocal, deviceHardware = hw, deviceLinks = deviceLinks, + hasDeviceModel = hasDeviceModel, reportedTarget = pioEnv, isManaged = identity.profile.config?.security?.is_managed ?: false, isFahrenheit = temperatureUnit == TemperatureUnit.FAHRENHEIT, @@ -273,6 +276,8 @@ constructor( private data class HardwareKey(val hwModel: Int, val target: String?) + private data class HardwareGroup(val hardware: DeviceHardware?, val links: List, val hasModel: Boolean) + private data class LogsGroup( val telemetry: List, val packets: List, diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/GlbBounds.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/GlbBounds.kt new file mode 100644 index 00000000000..8beeca76606 --- /dev/null +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/GlbBounds.kt @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.model + +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.float +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive + +/** Axis-aligned bounds of a model in its own units (meters for the flasher's models), Y up. */ +data class GlbBounds(val min: FloatArray, val max: FloatArray) { + val center: FloatArray + get() = FloatArray(AXES) { (min[it] + max[it]) / 2f } + + val extent: Float + get() = (0 until AXES).maxOf { max[it] - min[it] } + + override fun equals(other: Any?): Boolean = + this === other || (other is GlbBounds && min.contentEquals(other.min) && max.contentEquals(other.max)) + + override fun hashCode(): Int = 31 * min.contentHashCode() + max.contentHashCode() + + companion object { + private const val AXES = 3 + private const val HEADER_BYTES = 12 + private const val CHUNK_TYPE_OFFSET = 4 + private const val CHUNK_HEADER_BYTES = 8 + private const val GLB_MAGIC = 0x46546C67 // "glTF" + private const val JSON_CHUNK = 0x4E4F534A // "JSON" + private const val INT_BYTES = 4 + private const val BYTE_BITS = 8 + private const val BYTE_MASK = 0xFF + private val json = Json { ignoreUnknownKeys = true } + + /** + * The union of every mesh primitive's `POSITION` accessor bounds, which glTF requires each accessor to carry. + * Node transforms are ignored: the flasher's models bake theirs into the vertices. Null for anything that is + * not a GLB with at least one positioned primitive. + */ + fun parse(glb: ByteArray): GlbBounds? = jsonChunk(glb)?.let { chunk -> + runCatching { boundsOf(json.parseToJsonElement(chunk).jsonObject) }.getOrNull() + } + + /** The notice the model carries in glTF's `asset.copyright`, or null when it has none. */ + fun copyright(glb: ByteArray): String? = jsonChunk(glb)?.let { chunk -> + runCatching { + json + .parseToJsonElement(chunk) + .jsonObject["asset"] + ?.jsonObject + ?.get("copyright") + ?.jsonPrimitive + ?.content + } + .getOrNull() + ?.takeIf { it.isNotBlank() } + } + + private fun jsonChunk(glb: ByteArray): String? { + val start = HEADER_BYTES + CHUNK_HEADER_BYTES + if (glb.size < start) return null + val length = glb.readIntLe(HEADER_BYTES) + val valid = + glb.readIntLe(0) == GLB_MAGIC && + glb.readIntLe(HEADER_BYTES + CHUNK_TYPE_OFFSET) == JSON_CHUNK && + length >= 0 && + start + length <= glb.size + return if (valid) glb.decodeToString(start, start + length) else null + } + + /** + * Bounds of the display glass: primitives of a mesh named `screen`, or using a material named `screen`. Null + * when the model does not mark its display. + */ + fun screen(glb: ByteArray): GlbBounds? = jsonChunk(glb)?.let { chunk -> + runCatching { + val root = json.parseToJsonElement(chunk).jsonObject + val materials = + root["materials"]?.jsonArray.orEmpty().map { it.jsonObject["name"]?.jsonPrimitive?.content } + boundsOf(root) { mesh, primitive -> + mesh["name"]?.jsonPrimitive?.content.equals(SCREEN, ignoreCase = true) || + primitive["material"] + ?.jsonPrimitive + ?.int + ?.let { materials.getOrNull(it) } + .equals(SCREEN, ignoreCase = true) + } + } + .getOrNull() + } + + private const val SCREEN = "screen" + + private fun boundsOf( + root: JsonObject, + include: (mesh: JsonObject, primitive: JsonObject) -> Boolean = { _, _ -> true }, + ): GlbBounds? { + val accessors = root["accessors"]?.jsonArray.orEmpty() + val positions = + root["meshes"]?.jsonArray.orEmpty().flatMap { meshElement -> + val mesh = meshElement.jsonObject + mesh["primitives"] + ?.jsonArray + .orEmpty() + .filter { include(mesh, it.jsonObject) } + .mapNotNull { primitive -> + primitive.jsonObject["attributes"] + ?.jsonObject + ?.get("POSITION") + ?.jsonPrimitive + ?.int + ?.let { accessors[it].jsonObject } + ?.takeIf { "min" in it && "max" in it } + } + } + if (positions.isEmpty()) return null + fun axis(key: String, axis: Int) = positions.map { it.getValue(key).jsonArray[axis].jsonPrimitive.float } + return GlbBounds( + min = FloatArray(AXES) { axis("min", it).min() }, + max = FloatArray(AXES) { axis("max", it).max() }, + ) + } + + private fun ByteArray.readIntLe(offset: Int): Int = (0 until INT_BYTES).fold(0) { acc, i -> + acc or ((this[offset + i].toInt() and BYTE_MASK) shl (BYTE_BITS * i)) + } + } +} diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/MetricsState.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/MetricsState.kt index 931ca6526ec..dcd62eed046 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/MetricsState.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/MetricsState.kt @@ -51,6 +51,8 @@ data class MetricsState( val deviceHardware: DeviceHardware? = null, /** msh.to vendor/marketplace links for this device's hardware, region-filtered and sorted (vendor first). */ val deviceLinks: List = emptyList(), + /** Whether the flasher serves a 3D model for [deviceHardware]'s hero illustration. */ + val hasDeviceModel: Boolean = false, val firmwareEdition: FirmwareEdition? = null, val latestStableFirmware: FirmwareRelease = FirmwareRelease(), val latestAlphaFirmware: FirmwareRelease = FirmwareRelease(), diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/NodeDetailAction.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/NodeDetailAction.kt index 523ecc9d8b1..ffd6f56366e 100644 --- a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/NodeDetailAction.kt +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/NodeDetailAction.kt @@ -17,6 +17,7 @@ package org.meshtastic.feature.node.model import org.meshtastic.core.common.util.MeasurementSystem +import org.meshtastic.core.model.DeviceHardware import org.meshtastic.core.model.Node import org.meshtastic.core.navigation.Route import org.meshtastic.feature.node.component.NodeMenuAction @@ -36,6 +37,9 @@ sealed interface NodeDetailAction { data object ShareContact : NodeDetailAction + /** Open the real-size 3D model of the node's hardware. */ + data class OpenDeviceModel(val deviceHardware: DeviceHardware) : NodeDetailAction + // Opens the compass sheet scoped to a target node and the user’s preferred units. data class OpenCompass(val node: Node, val displayUnits: MeasurementSystem) : NodeDetailAction } diff --git a/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/RestPose.kt b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/RestPose.kt new file mode 100644 index 00000000000..546d370d37d --- /dev/null +++ b/feature/node/src/commonMain/kotlin/org/meshtastic/feature/node/model/RestPose.kt @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.model + +import kotlin.math.PI +import kotlin.math.cos +import kotlin.math.sin + +/** + * Which face of a model rests on the surface it is placed on. The flasher's models are authored Y up with the front + * facing +Z, so a pitch about X lays the device on its back or front and a roll about Z on a side. + */ +enum class RestPose(val pitchDegrees: Float, val rollDegrees: Float) { + Upright(0f, 0f), + FaceUp(-QUARTER_TURN, 0f), + FaceDown(QUARTER_TURN, 0f), + LeftSide(0f, QUARTER_TURN), + RightSide(0f, -QUARTER_TURN), + ; + + /** The model's bounds once rotated into this pose, before any offset. */ + fun rotatedBounds(bounds: GlbBounds): GlbBounds { + val ends = listOf(bounds.min, bounds.max) + val corners = ends.flatMap { x -> ends.flatMap { y -> ends.map { z -> rotate(x[0], y[1], z[2]) } } } + return GlbBounds( + min = FloatArray(AXES) { axis -> corners.minOf { it[axis] } }, + max = FloatArray(AXES) { axis -> corners.maxOf { it[axis] } }, + ) + } + + /** Where to put the rotated model so its lowest face touches y = 0 and its footprint centers on the origin. */ + fun offset(bounds: GlbBounds): FloatArray { + val rotated = rotatedBounds(bounds) + return floatArrayOf(-rotated.center[0], -rotated.min[1], -rotated.center[2]) + } + + private fun rotate(x: Float, y: Float, z: Float): FloatArray { + val pitch = pitchDegrees * DEGREES_TO_RADIANS + val roll = rollDegrees * DEGREES_TO_RADIANS + // Pitch about X, then roll about Z; a pose uses one or the other. + val y1 = y * cos(pitch) - z * sin(pitch) + val z1 = y * sin(pitch) + z * cos(pitch) + return floatArrayOf(x * cos(roll) - y1 * sin(roll), x * sin(roll) + y1 * cos(roll), z1) + } +} + +private const val QUARTER_TURN = 90f +private const val AXES = 3 +private const val DEGREES_TO_RADIANS = (PI / 180.0).toFloat() diff --git a/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/domain/usecase/TracerouteHistoryUseCaseTest.kt b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/domain/usecase/TracerouteHistoryUseCaseTest.kt index 07d61bd907b..aad87c0599d 100644 --- a/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/domain/usecase/TracerouteHistoryUseCaseTest.kt +++ b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/domain/usecase/TracerouteHistoryUseCaseTest.kt @@ -76,6 +76,7 @@ class TracerouteHistoryUseCaseTest { radioConfigRepository = FakeRadioConfigRepository(), deviceHardwareRepository = FakeDeviceHardwareRepository(), deviceLinkRepository = mock(MockMode.autofill), + deviceModelRepository = mock(MockMode.autofill), firmwareReleaseRepository = FakeFirmwareReleaseRepository(), nodeRequestActions = nodeRequestActions, localeUnitsProvider = FakeLocaleUnitsProvider(), diff --git a/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/GlbBoundsTest.kt b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/GlbBoundsTest.kt new file mode 100644 index 00000000000..4863cd9a55b --- /dev/null +++ b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/GlbBoundsTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.model + +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull + +class GlbBoundsTest { + + @Test + fun unionsEveryPrimitivesPositionAccessor() { + val glb = + glb( + """ + {"accessors":[ + {"min":[-0.0275,0,-0.00325],"max":[0.0275,0.0858,0.00325]}, + {"min":[0.01,0.08,-0.009],"max":[0.026,0.0935,0.009]}, + {"min":[-9,-9,-9],"max":[9,9,9]} + ], + "meshes":[ + {"primitives":[{"attributes":{"POSITION":0,"NORMAL":2}}]}, + {"primitives":[{"attributes":{"POSITION":1}}]} + ]} + """, + ) + + val bounds = assertNotNull(GlbBounds.parse(glb)) + + assertContentEquals(floatArrayOf(-0.0275f, 0f, -0.009f), bounds.min) + assertContentEquals(floatArrayOf(0.0275f, 0.0935f, 0.009f), bounds.max) + assertEquals(0.0935f, bounds.extent) + assertContentEquals(floatArrayOf(0f, 0.04675f, 0f), bounds.center) + } + + @Test + fun readsTheModelsOwnCopyrightNotice() { + val glb = glb("""{"asset":{"version":"2.0","copyright":"Copyright © 2026 Meshtastic LLC."}}""") + + assertEquals("Copyright © 2026 Meshtastic LLC.", GlbBounds.copyright(glb)) + } + + @Test + fun noCopyrightWhenTheModelCarriesNone() { + assertNull(GlbBounds.copyright(glb("""{"asset":{"version":"2.0"}}"""))) + assertNull(GlbBounds.copyright("not a model".encodeToByteArray())) + } + + @Test + fun screenIsTheMeshOrMaterialNamedScreen() { + val glb = + glb( + """ + {"accessors":[ + {"min":[-0.026,0,-0.015],"max":[0.026,0.138,0.017]}, + {"min":[-0.015,0.032,0.0158],"max":[0.015,0.047,0.016]}, + {"min":[0.02,0.1,0],"max":[0.021,0.11,0.001]} + ], + "materials":[{"name":"case"},{"name":"Screen"}], + "meshes":[ + {"name":"Body","primitives":[{"attributes":{"POSITION":0},"material":0}]}, + {"name":"Glass","primitives":[{"attributes":{"POSITION":1},"material":1}]}, + {"name":"screen","primitives":[{"attributes":{"POSITION":2},"material":0}]} + ]} + """, + ) + + val screen = assertNotNull(GlbBounds.screen(glb)) + + assertContentEquals(floatArrayOf(-0.015f, 0.032f, 0f), screen.min) + assertContentEquals(floatArrayOf(0.021f, 0.11f, 0.016f), screen.max) + } + + @Test + fun noScreenWhenNothingIsNamedScreen() { + val glb = + glb( + """ + {"accessors":[{"min":[0,0,0],"max":[1,1,1]}], + "meshes":[{"primitives":[{"attributes":{"POSITION":0}}]}]} + """, + ) + + assertNull(GlbBounds.screen(glb)) + } + + @Test + fun rejectsNonGlbBytes() { + assertNull(GlbBounds.parse("not a model".encodeToByteArray())) + assertNull(GlbBounds.parse(ByteArray(4))) + } + + @Test + fun rejectsModelWithoutPositions() { + assertNull(GlbBounds.parse(glb("""{"accessors":[],"meshes":[{"primitives":[{"attributes":{}}]}]}"""))) + } + + private fun glb(json: String): ByteArray { + val body = json.trimIndent().encodeToByteArray() + val header = ByteArray(20) + header.putIntLe(0, 0x46546C67) + header.putIntLe(4, 2) + header.putIntLe(8, header.size + body.size) + header.putIntLe(12, body.size) + header.putIntLe(16, 0x4E4F534A) + return header + body + } + + private fun ByteArray.putIntLe(offset: Int, value: Int) { + for (i in 0 until 4) this[offset + i] = (value shr (8 * i)).toByte() + } +} diff --git a/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/RestPoseTest.kt b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/RestPoseTest.kt new file mode 100644 index 00000000000..fd7f42f61d5 --- /dev/null +++ b/feature/node/src/commonTest/kotlin/org/meshtastic/feature/node/model/RestPoseTest.kt @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.model + +import kotlin.math.abs +import kotlin.test.Test +import kotlin.test.assertTrue + +class RestPoseTest { + // The WisMesh Pocket: 52 mm wide, 138 mm tall with its antenna, 32 mm deep, front facing +Z. + private val pocket = GlbBounds(floatArrayOf(-0.026f, 0f, -0.015f), floatArrayOf(0.026f, 0.1382f, 0.017f)) + + @Test + fun everyPoseRestsOnTheSurfaceCenteredOnTheAnchor() { + RestPose.entries.forEach { pose -> + val placed = placed(pose) + assertNear(0f, placed.min[1], "$pose bottom") + assertNear(0f, placed.center[0], "$pose x center") + assertNear(0f, placed.center[2], "$pose z center") + } + } + + @Test fun uprightStandsAtFullHeight() = assertNear(0.1382f, placed(RestPose.Upright).max[1], "height") + + @Test + fun faceUpAndFaceDownLieAsDeepAsTheCase() { + assertNear(0.032f, placed(RestPose.FaceUp).max[1], "face up height") + assertNear(0.032f, placed(RestPose.FaceDown).max[1], "face down height") + } + + @Test + fun aSideRestsAsWideAsTheCase() { + assertNear(0.052f, placed(RestPose.LeftSide).max[1], "left height") + assertNear(0.052f, placed(RestPose.RightSide).max[1], "right height") + } + + @Test + fun faceUpTurnsTheScreenToTheSkyAndFaceDownToTheSurface() { + val screen = GlbBounds(floatArrayOf(-0.015f, 0.032f, 0.0158f), floatArrayOf(0.015f, 0.047f, 0.016f)) + val up = RestPose.FaceUp + val down = RestPose.FaceDown + assertTrue(up.rotatedBounds(screen).center[1] > up.rotatedBounds(pocket).center[1], "face up") + assertTrue(down.rotatedBounds(screen).center[1] < down.rotatedBounds(pocket).center[1], "face down") + } + + private fun placed(pose: RestPose): GlbBounds { + val rotated = pose.rotatedBounds(pocket) + val offset = pose.offset(pocket) + return GlbBounds( + FloatArray(3) { rotated.min[it] + offset[it] }, + FloatArray(3) { rotated.max[it] + offset[it] }, + ) + } + + private fun assertNear(expected: Float, actual: Float, what: String) = + assertTrue(abs(expected - actual) < 1e-5f, "$what: expected $expected, was $actual") +} diff --git a/feature/node/src/iosMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.ios.kt b/feature/node/src/iosMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.ios.kt new file mode 100644 index 00000000000..b1f25f005ab --- /dev/null +++ b/feature/node/src/iosMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.ios.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.feature.node.model.GlbBounds + +@Composable +internal actual fun DeviceModelScene( + glb: ByteArray, + bounds: GlbBounds, + onError: () -> Unit, + modifier: Modifier, + visible: Boolean, + screen: GlbBounds?, + screenFrame: MirrorFrame?, +) { + if (visible) SceneViewerModelScene(glb, bounds, onError, modifier) +} diff --git a/feature/node/src/jvmMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.jvm.kt b/feature/node/src/jvmMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.jvm.kt new file mode 100644 index 00000000000..b1f25f005ab --- /dev/null +++ b/feature/node/src/jvmMain/kotlin/org/meshtastic/feature/node/component/SceneViewerPlatform.jvm.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.node.component + +import androidx.compose.runtime.Composable +import androidx.compose.ui.Modifier +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.feature.node.model.GlbBounds + +@Composable +internal actual fun DeviceModelScene( + glb: ByteArray, + bounds: GlbBounds, + onError: () -> Unit, + modifier: Modifier, + visible: Boolean, + screen: GlbBounds?, + screenFrame: MirrorFrame?, +) { + if (visible) SceneViewerModelScene(glb, bounds, onError, modifier) +} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt index 5af74080756..66390eeb52a 100644 --- a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/Debug.kt @@ -77,6 +77,7 @@ import org.meshtastic.core.resources.debug_panel import org.meshtastic.core.resources.debug_store_logs_summary import org.meshtastic.core.resources.debug_store_logs_title import org.meshtastic.core.resources.debug_tab_app_logs +import org.meshtastic.core.resources.debug_tab_mirror import org.meshtastic.core.resources.debug_tab_packets import org.meshtastic.core.resources.log_retention_days import org.meshtastic.core.resources.log_retention_days_quantity @@ -185,11 +186,20 @@ fun DebugScreen(onNavigateUp: () -> Unit, viewModel: DebugViewModel) { onClick = { selectedTab = 1 }, text = { Text(stringResource(Res.string.debug_tab_app_logs)) }, ) + Tab( + selected = selectedTab == 2, + onClick = { selectedTab = 2 }, + text = { Text(stringResource(Res.string.debug_tab_mirror)) }, + ) } if (selectedTab == 1) { LogcatContent(modifier = Modifier.fillMaxSize()) return@Column } + if (selectedTab == 2) { + DisplayMirrorContent(modifier = Modifier.fillMaxSize()) + return@Column + } LazyColumn(modifier = Modifier.fillMaxSize(), state = listState) { stickyHeader { val animatedAlpha by diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt new file mode 100644 index 00000000000..2006bbaafb8 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirror.kt @@ -0,0 +1,400 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.combinedClickable +import androidx.compose.foundation.gestures.detectDragGestures +import androidx.compose.foundation.gestures.detectTapGestures +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.imePadding +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.FilterChip +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.DisposableEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.focus.FocusRequester +import androidx.compose.ui.focus.onFocusChanged +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.hapticfeedback.HapticFeedbackType +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.platform.LocalHapticFeedback +import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import org.jetbrains.compose.resources.stringResource +import org.koin.compose.viewmodel.koinViewModel +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette +import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.mirror_active +import org.meshtastic.core.resources.mirror_back +import org.meshtastic.core.resources.mirror_eink_hint +import org.meshtastic.core.resources.mirror_frame_info +import org.meshtastic.core.resources.mirror_hint_keyboard_active +import org.meshtastic.core.resources.mirror_hint_tap_keyboard +import org.meshtastic.core.resources.mirror_hint_tap_touch +import org.meshtastic.core.resources.mirror_keyboard +import org.meshtastic.core.resources.mirror_no_frame +import org.meshtastic.core.resources.mirror_not_connected +import org.meshtastic.core.resources.mirror_off +import org.meshtastic.core.resources.refresh +import org.meshtastic.core.ui.input.RemoteKey +import org.meshtastic.core.ui.input.RemoteKeyHandler +import org.meshtastic.core.ui.input.RemoteKeyboardSink +import org.meshtastic.proto.DisplayInfo + +// M3 comfortable target for remote-control keys (48dp minimum + breathing room). +private val DPAD_KEY_SIZE = 56.dp + +// OS-typematic-style auto-repeat, slowed to what a LoRa radio UI can render. +internal const val REPEAT_INITIAL_DELAY_MS = 500L +internal const val REPEAT_INTERVAL_MS = 100L + +// Swipes on the mirror shorter than this are ignored as accidental. +private val SWIPE_THRESHOLD = 48.dp + +// Mirror + D-pad fit comfortably side by side above this content width. +private val SIDE_BY_SIDE_MIN_WIDTH = 760.dp + +/** Toggle, one-shot refresh, and the live frame's geometry. Split out to keep [DisplayMirrorContent] readable. */ +@Composable +private fun MirrorHeaderRow( + mirroring: Boolean, + active: Boolean, + frame: MirrorFrame?, + onToggle: (Boolean) -> Unit, + onRefresh: () -> Unit, +) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + Switch(checked = mirroring, onCheckedChange = onToggle, enabled = active) + Text( + text = stringResource(if (mirroring) Res.string.mirror_active else Res.string.mirror_off), + style = MaterialTheme.typography.titleMedium, + ) + OutlinedButton(onClick = onRefresh, enabled = active) { Text(stringResource(Res.string.refresh)) } + if (frame != null) { + Text( + text = stringResource(Res.string.mirror_frame_info, frame.width, frame.height, frame.frameId), + style = MaterialTheme.typography.labelSmall, + ) + } + } +} + +/** Live view of the connected device's screen, with remote D-pad, keyboard, and touch control. */ +@Composable +fun DisplayMirrorContent(modifier: Modifier = Modifier, viewModel: DisplayMirrorViewModel = koinViewModel()) { + val frame by viewModel.frame.collectAsStateWithLifecycle() + val palette by viewModel.palette.collectAsStateWithLifecycle() + val mirroring by viewModel.mirroring.collectAsStateWithLifecycle() + val connected by viewModel.connected.collectAsStateWithLifecycle() + val displayInfo by viewModel.displayInfo.collectAsStateWithLifecycle() + val active = connected && !viewModel.managed.collectAsStateWithLifecycle().value + + // Don't leave the device streaming to a hidden tab or abandoned screen. + DisposableEffect(Unit) { onDispose { viewModel.stopMirroring() } } + + Column( + modifier = modifier.fillMaxSize().imePadding().verticalScroll(rememberScrollState()).padding(16.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + val currentFrame = frame + MirrorHeaderRow( + mirroring = mirroring, + active = active, + frame = currentFrame, + onToggle = viewModel::setMirror, + onRefresh = viewModel::requestFrame, + ) + + if (displayInfo?.panel_class == DisplayInfo.PanelClass.EINK) { + Text(text = stringResource(Res.string.mirror_eink_hint), style = MaterialTheme.typography.labelSmall) + } + + when { + !connected -> { + Text(text = stringResource(Res.string.mirror_not_connected)) + DpadCluster(enabled = false, onEvent = viewModel::sendKey) + } + + currentFrame == null -> { + Text(text = stringResource(Res.string.mirror_no_frame)) + DpadCluster(enabled = active, onEvent = viewModel::sendKey) + } + + else -> + MirrorWithControls( + currentFrame, + palette, + enabled = active, + hasTouch = displayInfo?.has_touch == true, + onEvent = viewModel::sendKey, + onTouch = viewModel::sendTouch, + onChar = viewModel::sendChar, + ) + } + } +} + +/** Controls sit beside the mirror when the window is wide enough, below it otherwise. */ +@Suppress("LongParameterList") +@Composable +private fun MirrorWithControls( + frame: MirrorFrame, + palette: MirrorPalette?, + enabled: Boolean, + hasTouch: Boolean, + onEvent: (Int) -> Unit, + onTouch: (Int, Int, Int) -> Unit, + onChar: (Int) -> Unit, +) { + BoxWithConstraints(modifier = Modifier.fillMaxWidth()) { + if (maxWidth >= SIDE_BY_SIDE_MIN_WIDTH) { + Row( + horizontalArrangement = Arrangement.spacedBy(24.dp, Alignment.CenterHorizontally), + verticalAlignment = Alignment.CenterVertically, + modifier = Modifier.fillMaxWidth(), + ) { + MirrorSurface( + frame, + palette, + hasTouch, + onEvent, + onTouch, + onChar, + modifier = Modifier.weight(1f, fill = false), + ) + DpadCluster(enabled = enabled, onEvent = onEvent) + } + } else { + Column( + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(12.dp), + modifier = Modifier.fillMaxWidth(), + ) { + MirrorSurface(frame, palette, hasTouch, onEvent, onTouch, onChar) + DpadCluster(enabled = enabled, onEvent = onEvent) + } + } + } +} + +/** + * The live mirror plus its direct input affordances: a Keyboard chip toggles key capture, which raises the soft + * keyboard (arrows navigate, Enter = OK, Esc = Back, typed text reaches the device as characters), swiping in a + * cardinal direction sends one direction event, and on touch-capable devices tapping or long-pressing the image + * forwards real touch coordinates; elsewhere a tap toggles capture. + */ +@Composable +@Suppress("LongParameterList") +private fun MirrorSurface( + frame: MirrorFrame, + palette: MirrorPalette?, + hasTouch: Boolean, + onEvent: (Int) -> Unit, + onTouch: (Int, Int, Int) -> Unit, + onChar: (Int) -> Unit, + modifier: Modifier = Modifier, +) { + val focusRequester = remember { FocusRequester() } + val focusManager = LocalFocusManager.current + var focused by remember { mutableStateOf(false) } + val currentFrame by rememberUpdatedState(frame) + val focusColor = MaterialTheme.colorScheme.primary + + fun toggleKeyboard() = if (focused) focusManager.clearFocus() else focusRequester.requestFocus() + + Column( + modifier = modifier, + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(4.dp), + ) { + Box( + modifier = + Modifier.pointerInput(hasTouch) { + detectTapGestures( + onTap = { offset -> + if (hasTouch) { + onTouch( + INPUT_USER_PRESS, + offset.toDeviceX(size.width, currentFrame), + offset.toDeviceY(size.height, currentFrame), + ) + } else { + // Tap toggles keyboard control so there is always a way out of capture. + toggleKeyboard() + } + }, + onLongPress = { offset -> + // Physical touch drivers map a long-press to SELECT with coordinates. + if (hasTouch) { + onTouch( + INPUT_SELECT, + offset.toDeviceX(size.width, currentFrame), + offset.toDeviceY(size.height, currentFrame), + ) + } + }, + ) + } + .swipeToDirection(onEvent) + .border(width = 2.dp, color = if (focused) focusColor else Color.Transparent), + ) { + MirrorFrameImage(frame, palette) + RemoteKeyboardSink( + focusRequester = focusRequester, + handler = mirrorKeyHandler(onEvent, onChar), + modifier = Modifier.onFocusChanged { focused = it.isFocused }, + ) + } + MirrorControlHints(focused = focused, hasTouch = hasTouch, onToggleKeyboard = { toggleKeyboard() }) + } +} + +@Composable +private fun MirrorControlHints(focused: Boolean, hasTouch: Boolean, onToggleKeyboard: () -> Unit) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(8.dp)) { + FilterChip( + selected = focused, + onClick = onToggleKeyboard, + label = { Text(stringResource(Res.string.mirror_keyboard)) }, + ) + Text( + text = + stringResource( + when { + focused -> Res.string.mirror_hint_keyboard_active + hasTouch -> Res.string.mirror_hint_tap_touch + else -> Res.string.mirror_hint_tap_keyboard + }, + ), + style = MaterialTheme.typography.labelSmall, + ) + } +} + +/** + * Maps the keyboard sink onto device input: navigation keys become input events, typed text and Backspace/Tab become + * characters. Ctrl/Alt chords are left to the host, so desktop shortcuts never reach the device. + */ +private fun mirrorKeyHandler(onEvent: (Int) -> Unit, onChar: (Int) -> Unit) = RemoteKeyHandler( + onText = { text -> forEachCodePoint(text, onChar) }, + onEnter = { onEvent(INPUT_SELECT) }, + onBackspace = { onChar(CHAR_BACKSPACE) }, + onKey = { key -> + when (key) { + RemoteKey.TAB -> onChar(CHAR_TAB) + else -> key.toInputEvent()?.let(onEvent) + } + }, +) + +/** Converts a completed drag into one direction event along its dominant axis, ignoring short accidental swipes. */ +private fun Modifier.swipeToDirection(onEvent: (Int) -> Unit): Modifier = pointerInput(Unit) { + val threshold = SWIPE_THRESHOLD.toPx() + var drag = Offset.Zero + detectDragGestures( + onDragStart = { drag = Offset.Zero }, + onDrag = { change, amount -> + change.consume() + drag += amount + }, + onDragEnd = { + val horizontal = drag.x + val vertical = drag.y + when { + kotlin.math.abs(horizontal) < threshold && kotlin.math.abs(vertical) < threshold -> Unit + + kotlin.math.abs(horizontal) >= kotlin.math.abs(vertical) -> + onEvent(if (horizontal > 0) INPUT_RIGHT else INPUT_LEFT) + + else -> onEvent(if (vertical > 0) INPUT_DOWN else INPUT_UP) + } + }, + ) +} + +/** The remote cluster: circular 5-way ring (see [DpadRing]) with Back below-left, per TV-remote convention. */ +@Composable +private fun DpadCluster(enabled: Boolean, onEvent: (Int) -> Unit, modifier: Modifier = Modifier) { + Column( + modifier = modifier, + horizontalAlignment = Alignment.Start, + verticalArrangement = Arrangement.spacedBy(12.dp), + ) { + DpadRing( + enabled = enabled, + onEvent = onEvent, + onSelect = { onEvent(INPUT_SELECT) }, + onSelectLong = { onEvent(INPUT_SELECT_LONG) }, + ) + BackKey(enabled = enabled) { onEvent(INPUT_BACK) } + } +} + +@Composable +private fun BackKey(enabled: Boolean, onBack: () -> Unit) { + val haptics = LocalHapticFeedback.current + Box( + contentAlignment = Alignment.Center, + modifier = + Modifier.size(DPAD_KEY_SIZE) + .clip(CircleShape) + .background( + if (enabled) { + MaterialTheme.colorScheme.secondaryContainer + } else { + MaterialTheme.colorScheme.surfaceVariant + }, + ) + .combinedClickable(enabled = enabled) { + haptics.performHapticFeedback(HapticFeedbackType.VirtualKey) + onBack() + }, + ) { + Text( + text = stringResource(Res.string.mirror_back), + style = MaterialTheme.typography.labelMedium, + color = dpadContentColor(enabled), + ) + } +} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorUtils.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorUtils.kt new file mode 100644 index 00000000000..bcfa1e1b700 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorUtils.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.material3.MaterialTheme +import androidx.compose.runtime.Composable +import androidx.compose.runtime.ReadOnlyComposable +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Color +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.ui.input.RemoteKey + +// Firmware input_broker_event codes (src/input/InputBroker.h). USER_PRESS is +// what physical touch drivers emit for a tap, with touch coordinates attached. +internal const val INPUT_SELECT = 10 +internal const val INPUT_SELECT_LONG = 11 +internal const val INPUT_UP = 17 +internal const val INPUT_DOWN = 18 +internal const val INPUT_LEFT = 19 +internal const val INPUT_RIGHT = 20 +internal const val INPUT_BACK = 27 +internal const val INPUT_USER_PRESS = 28 + +// ANYKEY carries a character in kb_char rather than a navigation code. +internal const val INPUT_ANYKEY = 0xFF + +// LVGL's backspace key value; typed as a character so text fields delete +// rather than navigating back (which is what Esc is for). +internal const val CHAR_BACKSPACE = 8 + +// LVGL's next-focus key value, typed as a character like Backspace. +internal const val CHAR_TAB = 9 + +/** Scales a tap position on the scaled-up mirror image back to panel pixel coordinates. */ +internal fun Offset.toDeviceX(boxWidthPx: Int, frame: MirrorFrame): Int = + (x / boxWidthPx * frame.width).toInt().coerceIn(0, frame.width - 1) + +internal fun Offset.toDeviceY(boxHeightPx: Int, frame: MirrorFrame): Int = + (y / boxHeightPx * frame.height).toInt().coerceIn(0, frame.height - 1) + +/** The device navigation event for a key the mirror forwards; null for keys a device UI has no use for. */ +internal fun RemoteKey.toInputEvent(): Int? = when (this) { + RemoteKey.UP -> INPUT_UP + RemoteKey.DOWN -> INPUT_DOWN + RemoteKey.LEFT -> INPUT_LEFT + RemoteKey.RIGHT -> INPUT_RIGHT + RemoteKey.ESCAPE -> INPUT_BACK + else -> null +} + +/** Calls [action] with each Unicode code point of [text], joining surrogate pairs so one character is one kb_char. */ +internal fun forEachCodePoint(text: String, action: (Int) -> Unit) { + var i = 0 + while (i < text.length) { + val c = text[i] + val next = text.getOrNull(i + 1) + if (c.isHighSurrogate() && next != null && next.isLowSurrogate()) { + action( + SUPPLEMENTARY_BASE + + ((c.code - HIGH_SURROGATE_BASE) shl SURROGATE_SHIFT) + + (next.code - LOW_SURROGATE_BASE), + ) + i += 2 + } else { + action(c.code) + i++ + } + } +} + +private const val SUPPLEMENTARY_BASE = 0x10000 +private const val HIGH_SURROGATE_BASE = 0xD800 +private const val LOW_SURROGATE_BASE = 0xDC00 +private const val SURROGATE_SHIFT = 10 + +@Composable +@ReadOnlyComposable +internal fun dpadContentColor(enabled: Boolean): Color = + if (enabled) MaterialTheme.colorScheme.onSecondaryContainer else MaterialTheme.colorScheme.onSurfaceVariant diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorViewModel.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorViewModel.kt new file mode 100644 index 00000000000..b3ae14090b6 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorViewModel.kt @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.lifecycle.ViewModel +import androidx.lifecycle.viewModelScope +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.SharingStarted +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.stateIn +import kotlinx.coroutines.launch +import org.koin.core.annotation.KoinViewModel +import org.meshtastic.core.model.ConnectionState +import org.meshtastic.core.repository.AdminController +import org.meshtastic.core.repository.ConnectionStateProvider +import org.meshtastic.core.repository.DisplayMirrorManager +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette +import org.meshtastic.core.repository.NodeRepository +import org.meshtastic.core.repository.RadioConfigRepository +import org.meshtastic.proto.DisplayInfo + +@KoinViewModel +class DisplayMirrorViewModel( + private val displayMirrorManager: DisplayMirrorManager, + connectionStateProvider: ConnectionStateProvider, + nodeRepository: NodeRepository, + private val adminController: AdminController, + radioConfigRepository: RadioConfigRepository, +) : ViewModel() { + + val frame: StateFlow = displayMirrorManager.frame + + val palette: StateFlow = displayMirrorManager.palette + + val connected: StateFlow = + connectionStateProvider.connectionState + .map { it is ConnectionState.Connected } + .stateIn(viewModelScope, SharingStarted.Eagerly, false) + + /** Panel description from the connect handshake; null on display-less nodes and pre-DisplayInfo firmware. */ + val displayInfo: StateFlow = + nodeRepository.ourNodeInfo.map { it?.metadata?.display }.stateIn(viewModelScope, SharingStarted.Eagerly, null) + + /** + * Managed mode locks the device out of configuration, and the mirror is not a read-only view: arming it is an admin + * write and the controls inject input. The Debug Panel route it lives on is deliberately exempt from the gate every + * other settings route carries, so the restriction has to be enforced here. + */ + val managed: StateFlow = + radioConfigRepository.localConfigFlow + .map { it.security?.is_managed == true } + .stateIn(viewModelScope, SharingStarted.Eagerly, false) + + private val _mirroring = MutableStateFlow(false) + val mirroring: StateFlow = _mirroring.asStateFlow() + + init { + // A fresh mirror UI must never show a previous device's final screen as if live. + displayMirrorManager.reset() + // The device forgets the (non-persisted) mirror setting on disconnect/reboot; + // mirror the reset locally so the toggle never claims a dead stream is live. + viewModelScope.launch { + connected.collect { + if (!it) { + _mirroring.value = false + displayMirrorManager.reset() + } + } + } + // Managed mode can arrive mid-session with a stream already live; tear it down when it does. + viewModelScope.launch { managed.collect { if (it && _mirroring.value) setMirror(false) } } + } + + fun setMirror(enabled: Boolean) { + // Disarming is always allowed - it is how a live stream is torn down when managed mode arrives. + if (enabled && managed.value) return + adminController.setDisplayMirror(enabled) + _mirroring.value = enabled + } + + fun requestFrame() { + if (managed.value) return + adminController.requestDisplayFrame() + } + + fun sendKey(eventCode: Int) { + if (managed.value) return + adminController.sendInputEvent(eventCode) + } + + /** Forwards a typed character; the device UI receives it as a key press rather than a navigation event. */ + fun sendChar(codePoint: Int) { + if (managed.value) return + adminController.sendInputEvent(INPUT_ANYKEY, kbChar = codePoint) + } + + /** Forwards a tap/long-press on the mirrored image as a device touch event with panel coordinates. */ + fun sendTouch(eventCode: Int, x: Int, y: Int) { + if (managed.value) return + adminController.sendInputEvent(eventCode, touchX = x, touchY = y) + } + + /** Stops a live stream when the mirror UI goes away; safe to call redundantly. */ + fun stopMirroring() { + if (_mirroring.value) setMirror(false) + } + + override fun onCleared() = stopMirroring() +} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DpadRing.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DpadRing.kt new file mode 100644 index 00000000000..3895d7650d0 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/DpadRing.kt @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.ExperimentalFoundationApi +import androidx.compose.foundation.background +import androidx.compose.foundation.combinedClickable +import androidx.compose.foundation.gestures.awaitEachGesture +import androidx.compose.foundation.gestures.awaitFirstDown +import androidx.compose.foundation.gestures.waitForUpOrCancellation +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.offset +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.DrawScope +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.hapticfeedback.HapticFeedback +import androidx.compose.ui.hapticfeedback.HapticFeedbackType +import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.platform.LocalHapticFeedback +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.semantics.contentDescription +import androidx.compose.ui.semantics.onClick +import androidx.compose.ui.semantics.role +import androidx.compose.ui.semantics.semantics +import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.dp +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import org.jetbrains.compose.resources.StringResource +import org.jetbrains.compose.resources.stringResource +import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.mirror_key_down +import org.meshtastic.core.resources.mirror_key_left +import org.meshtastic.core.resources.mirror_key_ok +import org.meshtastic.core.resources.mirror_key_right +import org.meshtastic.core.resources.mirror_key_up +import org.meshtastic.core.ui.icon.KeyboardArrowDown +import org.meshtastic.core.ui.icon.KeyboardArrowLeft +import org.meshtastic.core.ui.icon.KeyboardArrowRight +import org.meshtastic.core.ui.icon.KeyboardArrowUp +import org.meshtastic.core.ui.icon.MeshtasticIcons +import kotlin.math.PI +import kotlin.math.atan2 +import kotlin.math.hypot + +// Geometry per the TV-remote research: ~220dp ring, 72-80dp center disc, +// a radial dead band between disc and ring so ambiguous presses do nothing. +private val RING_DIAMETER = 220.dp +private val OK_DISC_DIAMETER = 76.dp +private val DEAD_ZONE = 12.dp + +private const val WEDGE_SWEEP_DEG = 90f +private const val FULL_CIRCLE_DEG = 360f +private const val RAD_TO_DEG = (180.0 / PI).toFloat() + +// Minimum comfortable assistive-tech target (M3). +private val SEMANTICS_TARGET = 48.dp + +// One direction wedge of the ring: hit-test bounds, highlight arc, icon placement. +private class Wedge( + val eventCode: Int, + val label: StringResource, + val startAngleDeg: Float, + val iconOffsetX: Dp, + val iconOffsetY: Dp, +) + +@Suppress("MagicNumber") +private fun buildWedges(iconRadius: Dp): List = listOf( + Wedge(INPUT_RIGHT, Res.string.mirror_key_right, -45f, iconRadius, 0.dp), + Wedge(INPUT_DOWN, Res.string.mirror_key_down, 45f, 0.dp, iconRadius), + Wedge(INPUT_LEFT, Res.string.mirror_key_left, 135f, -iconRadius, 0.dp), + Wedge(INPUT_UP, Res.string.mirror_key_up, 225f, 0.dp, -iconRadius), +) + +@Composable +private fun wedgeIcon(eventCode: Int) = when (eventCode) { + INPUT_UP -> MeshtasticIcons.KeyboardArrowUp + INPUT_DOWN -> MeshtasticIcons.KeyboardArrowDown + INPUT_LEFT -> MeshtasticIcons.KeyboardArrowLeft + else -> MeshtasticIcons.KeyboardArrowRight +} + +private fun List.at(angleDeg: Float): Wedge = first { wedge -> + val start = (wedge.startAngleDeg + FULL_CIRCLE_DEG) % FULL_CIRCLE_DEG + val normalized = if (angleDeg < start) angleDeg + FULL_CIRCLE_DEG else angleDeg + normalized >= start && normalized < start + WEDGE_SWEEP_DEG +} + +/** + * Circular 5-way remote cluster: four direction wedges around a center OK disc, the layout every TV remote converged + * on. Directions fire on press and auto-repeat while held; OK taps SELECT and long-presses SELECT_LONG. Wedge + * hit-testing is by angle with a dead band around the disc; each wedge also carries its own semantics node so screen + * readers see four buttons, not one blob. + */ +@Composable +internal fun DpadRing( + enabled: Boolean, + onEvent: (Int) -> Unit, + onSelect: () -> Unit, + onSelectLong: () -> Unit, + modifier: Modifier = Modifier, +) { + val haptics = LocalHapticFeedback.current + var pressedWedge by remember { mutableStateOf(null) } + // Icons and semantics targets sit on the ring stroke's center line. + val wedges = remember { buildWedges((RING_DIAMETER / 2 + OK_DISC_DIAMETER / 2 + DEAD_ZONE) / 2) } + + val ringColor = + if (enabled) MaterialTheme.colorScheme.secondaryContainer else MaterialTheme.colorScheme.surfaceVariant + val pressedColor = MaterialTheme.colorScheme.primaryContainer + + Box(modifier = modifier.size(RING_DIAMETER), contentAlignment = Alignment.Center) { + Canvas( + modifier = + Modifier.size(RING_DIAMETER) + .ringPresses(enabled, wedges, haptics, onPressChange = { pressedWedge = it }, onEvent = onEvent), + ) { + drawRing(ringColor, pressedColor, wedges, pressedWedge) + } + WedgeDecorations(wedges = wedges, enabled = enabled, onEvent = onEvent) + OkDisc(enabled = enabled, haptics = haptics, onSelect = onSelect, onSelectLong = onSelectLong) + } +} + +/** Press handling for the ring: angle hit-testing outside the dead band, then fire + auto-repeat until release. */ +private fun Modifier.ringPresses( + enabled: Boolean, + wedges: List, + haptics: HapticFeedback, + onPressChange: (Int?) -> Unit, + onEvent: (Int) -> Unit, +): Modifier = pointerInput(enabled) { + if (!enabled) return@pointerInput + val outerR = size.width / 2f + val innerR = (OK_DISC_DIAMETER.toPx() / 2f) + DEAD_ZONE.toPx() + val center = Offset(outerR, outerR) + coroutineScope { + awaitEachGesture { + val down = awaitFirstDown() + val d = down.position - center + val r = hypot(d.x, d.y) + if (r < innerR || r > outerR) return@awaitEachGesture + down.consume() + val angle = (atan2(d.y, d.x) * RAD_TO_DEG + FULL_CIRCLE_DEG) % FULL_CIRCLE_DEG + val wedge = wedges.at(angle) + onPressChange(wedge.eventCode) + haptics.performHapticFeedback(HapticFeedbackType.VirtualKey) + onEvent(wedge.eventCode) + val repeater = launch { + delay(REPEAT_INITIAL_DELAY_MS) + while (isActive) { + haptics.performHapticFeedback(HapticFeedbackType.SegmentFrequentTick) + onEvent(wedge.eventCode) + delay(REPEAT_INTERVAL_MS) + } + } + try { + waitForUpOrCancellation() + } finally { + // The pointerInput coroutine can be cancelled mid-press (enabled + // flip, node removal); never leave a wedge stuck highlighted. + repeater.cancel() + onPressChange(null) + } + } + } +} + +private fun DrawScope.drawRing(ringColor: Color, pressedColor: Color, wedges: List, pressedWedge: Int?) { + val outerR = size.width / 2f + val visualInnerR = (OK_DISC_DIAMETER.toPx() / 2f) + DEAD_ZONE.toPx() + val strokeWidth = outerR - visualInnerR + val strokeRadius = (outerR + visualInnerR) / 2f + drawCircle(color = ringColor, radius = strokeRadius, style = Stroke(width = strokeWidth)) + pressedWedge?.let { pressed -> + val wedge = wedges.first { it.eventCode == pressed } + drawArc( + color = pressedColor, + startAngle = wedge.startAngleDeg, + sweepAngle = WEDGE_SWEEP_DEG, + useCenter = false, + topLeft = Offset(outerR - strokeRadius, outerR - strokeRadius), + size = Size(strokeRadius * 2f, strokeRadius * 2f), + style = Stroke(width = strokeWidth), + ) + } +} + +/** Chevron icons plus one invisible semantics button per wedge so assistive tech sees four discrete controls. */ +@Composable +private fun WedgeDecorations(wedges: List, enabled: Boolean, onEvent: (Int) -> Unit) { + val iconColor = + if (enabled) MaterialTheme.colorScheme.onSecondaryContainer else MaterialTheme.colorScheme.onSurfaceVariant + wedges.forEach { wedge -> + val label = stringResource(wedge.label) + Icon( + imageVector = wedgeIcon(wedge.eventCode), + contentDescription = null, + tint = iconColor, + modifier = Modifier.offset(x = wedge.iconOffsetX, y = wedge.iconOffsetY), + ) + Box( + modifier = + Modifier.offset(x = wedge.iconOffsetX, y = wedge.iconOffsetY).size(SEMANTICS_TARGET).semantics { + role = Role.Button + contentDescription = label + onClick(label = label) { + if (enabled) onEvent(wedge.eventCode) + enabled + } + }, + ) + } +} + +@OptIn(ExperimentalFoundationApi::class) +@Composable +private fun OkDisc(enabled: Boolean, haptics: HapticFeedback, onSelect: () -> Unit, onSelectLong: () -> Unit) { + Box( + contentAlignment = Alignment.Center, + modifier = + Modifier.size(OK_DISC_DIAMETER) + .clip(CircleShape) + .background( + if (enabled) MaterialTheme.colorScheme.primary else MaterialTheme.colorScheme.surfaceVariant, + ) + .combinedClickable( + enabled = enabled, + onLongClick = onSelectLong, + onClick = { + haptics.performHapticFeedback(HapticFeedbackType.VirtualKey) + onSelect() + }, + ), + ) { + val okColor = if (enabled) MaterialTheme.colorScheme.onPrimary else MaterialTheme.colorScheme.onSurfaceVariant + Text( + text = stringResource(Res.string.mirror_key_ok), + style = MaterialTheme.typography.titleMedium, + color = okColor, + ) + } +} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt new file mode 100644 index 00000000000..6e2826c09c6 --- /dev/null +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/debugging/MirrorFrameRenderer.kt @@ -0,0 +1,223 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.aspectRatio +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.widthIn +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.produceState +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Canvas +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.FilterQuality +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.drawscope.CanvasDrawScope +import androidx.compose.ui.unit.Density +import androidx.compose.ui.unit.LayoutDirection +import androidx.compose.ui.unit.dp +import kotlinx.coroutines.withContext +import org.jetbrains.compose.resources.stringResource +import org.koin.compose.koinInject +import org.meshtastic.core.di.CoroutineDispatchers +import org.meshtastic.core.repository.MirrorFormat +import org.meshtastic.core.repository.MirrorFrame +import org.meshtastic.core.repository.MirrorPalette +import org.meshtastic.core.resources.Res +import org.meshtastic.core.resources.mirror_device_screen + +// 4x scale for the common 128px-wide OLED; caps the image so the D-pad stays above the fold on desktop. +private val MAX_CANVAS_WIDTH = 512.dp + +// MONO_VLSB packs 8 vertically adjacent pixels per byte (one "page" row). +private const val PIXELS_PER_PAGE = 8 + +/** + * Renders a device frame — 1bpp MONO_VLSB or RGB565 — once per frame into a 1:1 [ImageBitmap] and scales it up with + * nearest-neighbor filtering — crisp device pixels, no fractional-scale seams, one pixel walk per frame instead of per + * recomposition. + */ +@Composable +internal fun MirrorFrameImage(frame: MirrorFrame, palette: MirrorPalette?, modifier: Modifier = Modifier) { + // A palette only applies when the frame references it; otherwise render monochrome. + val activePalette = palette?.takeIf { frame.paletteSignature != 0 && it.signature == frame.paletteSignature } + // Rendered off the main thread: an 800x480 frame is tens of milliseconds. The last bitmap stays up meanwhile. + val dispatchers = koinInject() + val bitmap by + produceState(null, frame, activePalette) { + value = withContext(dispatchers.default) { renderFrame(frame, activePalette) } + } + val sizing = + modifier + .widthIn(max = MAX_CANVAS_WIDTH) + .fillMaxWidth() + .aspectRatio(frame.width.toFloat() / frame.height.toFloat()) + bitmap?.let { + Image( + bitmap = it, + contentDescription = stringResource(Res.string.mirror_device_screen), + modifier = sizing, + filterQuality = FilterQuality.None, + ) + } ?: Box(sizing) +} + +/** One colorized rectangle with palette colors pre-resolved to Compose colors. */ +internal class ResolvedRegion( + val left: Int, + val top: Int, + val right: Int, + val bottom: Int, + val on: Color, + val off: Color, +) + +// Sentinel for "no run started"; 0 is black, which is the skipped background. +private const val NO_RUN = -1 + +/** Reads one little-endian RGB565 value from a packed pixel row. */ +@Suppress("MagicNumber") +private fun leRgb565At(pixels: ByteArray, index: Int): Int = + (pixels[index].toInt() and 0xFF) or ((pixels[index + 1].toInt() and 0xFF) shl 8) + +/** Expands RGB565 to 8-bit channels by bit replication (the canonical expansion; plain scaling floors). */ +@Suppress("MagicNumber") +internal fun rgb565ToColor(v: Int): Color { + val r5 = (v ushr 11) and 0x1F + val g6 = (v ushr 5) and 0x3F + val b5 = v and 0x1F + return Color(red = (r5 shl 3) or (r5 ushr 2), green = (g6 shl 2) or (g6 ushr 4), blue = (b5 shl 3) or (b5 ushr 2)) +} + +/** + * Resolves one pixel against the regions overlapping its row, highest table index winning — the firmware's precedence. + * [rowRegions] must already be culled to the pixel's row. + */ +internal fun resolvePixelColor( + x: Int, + set: Boolean, + rowRegions: List, + defaultOn: Color, + defaultOff: Color, +): Color { + for (i in rowRegions.indices.reversed()) { + val region = rowRegions[i] + if (x >= region.left && x < region.right) return if (set) region.on else region.off + } + return if (set) defaultOn else defaultOff +} + +internal fun MirrorPalette.resolveRegions(): List = regions.map { + ResolvedRegion( + left = it.x, + top = it.y, + right = it.x + it.width, + bottom = it.y + it.height, + on = rgb565ToColor(it.on_color), + off = rgb565ToColor(it.off_color), + ) +} + +private fun renderFrame(frame: MirrorFrame, palette: MirrorPalette?): ImageBitmap = when (frame.format) { + MirrorFormat.RGB565 -> renderRgb565Frame(frame) + MirrorFormat.MONO_VLSB -> renderMonoFrame(frame, palette) +} + +/** True-color frames (LVGL/MUI devices): little-endian RGB565, drawn as horizontal same-color runs. */ +private fun renderRgb565Frame(frame: MirrorFrame): ImageBitmap { + val bitmap = ImageBitmap(frame.width, frame.height) + val size = Size(frame.width.toFloat(), frame.height.toFloat()) + CanvasDrawScope().draw(Density(1f), LayoutDirection.Ltr, Canvas(bitmap), size) { + drawRect(color = Color.Black) + for (y in 0 until frame.height) { + val rowBase = y * frame.width * 2 + var runStart = 0 + // Track the raw RGB565 value: Color is a value class, so a nullable + // one would box on every pixel comparison. + var runValue = NO_RUN + fun flush(endExclusive: Int) { + if (runValue > 0) { + drawRect( + color = rgb565ToColor(runValue), + topLeft = Offset(runStart.toFloat(), y.toFloat()), + size = Size((endExclusive - runStart).toFloat(), 1f), + ) + } + } + for (x in 0 until frame.width) { + val value = leRgb565At(frame.pixels, rowBase + x * 2) + if (value != runValue) { + flush(x) + runStart = x + runValue = value + } + } + flush(frame.width) + } + } + return bitmap +} + +private fun renderMonoFrame(frame: MirrorFrame, palette: MirrorPalette?): ImageBitmap { + val defaultOn = palette?.let { rgb565ToColor(it.defaultOnColor) } ?: Color.White + val defaultOff = palette?.let { rgb565ToColor(it.defaultOffColor) } ?: Color.Black + val regions = palette?.resolveRegions().orEmpty() + + val bitmap = ImageBitmap(frame.width, frame.height) + val size = Size(frame.width.toFloat(), frame.height.toFloat()) + CanvasDrawScope().draw(Density(1f), LayoutDirection.Ltr, Canvas(bitmap), size) { + drawRect(color = defaultOff) + val rowRegions = ArrayList(regions.size) + for (y in 0 until frame.height) { + rowRegions.clear() + for (region in regions) if (y >= region.top && y < region.bottom) rowRegions.add(region) + + val page = (y / PIXELS_PER_PAGE) * frame.width + val bit = 1 shl (y % PIXELS_PER_PAGE) + // Emit horizontal runs of identical color; framebuffers are run-heavy, + // and one draw per run beats one draw per pixel by orders of magnitude. + var runStart = 0 + var runColor: Color? = null + fun flush(endExclusive: Int) { + val color = runColor + if (color != null && color != defaultOff) { + drawRect( + color = color, + topLeft = Offset(runStart.toFloat(), y.toFloat()), + size = Size((endExclusive - runStart).toFloat(), 1f), + ) + } + } + for (x in 0 until frame.width) { + val set = frame.pixels[page + x].toInt() and bit != 0 + val color = resolvePixelColor(x, set, rowRegions, defaultOn, defaultOff) + if (color != runColor) { + flush(x) + runStart = x + runColor = color + } + } + flush(frame.width) + } + } + return bitmap +} diff --git a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/radio/RadioConfig.kt b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/radio/RadioConfig.kt index b0036a16917..6550f061be6 100644 --- a/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/radio/RadioConfig.kt +++ b/feature/settings/src/commonMain/kotlin/org/meshtastic/feature/settings/radio/RadioConfig.kt @@ -274,8 +274,8 @@ private fun AdvancedSection(isManaged: Boolean, isOtaCapable: Boolean, enabled: onClick = { onNavigate(DiscoveryRoute.DiscoveryGraph) }, ) - // Always enabled: the Debug Panel reads app-local logs only — no radio connection, - // pending config response, or managed-mode restriction applies to it. + // Always enabled so logs stay reachable with no radio attached. Anything on this route that + // is NOT read-only must gate itself: it does not inherit `enabled` (see DisplayMirrorViewModel). ListItem( text = stringResource(Res.string.debug_panel), leadingIcon = MeshtasticIcons.BugReport, diff --git a/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorInputTest.kt b/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorInputTest.kt new file mode 100644 index 00000000000..70d71b76fa0 --- /dev/null +++ b/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/DisplayMirrorInputTest.kt @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import org.meshtastic.core.ui.input.RemoteKey +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class DisplayMirrorInputTest { + + private fun codePoints(text: String): List = mutableListOf().also { out -> + forEachCodePoint(text) { out += it } + } + + @Test + fun plainTextIsOneCodePointPerCharacter() { + assertEquals(listOf('h'.code, 'i'.code, ' '.code), codePoints("hi ")) + } + + @Test + fun aSurrogatePairIsOneCodePoint() { + assertEquals(listOf(0x1F600), codePoints("😀")) + assertEquals(listOf('a'.code, 0x1F600, 'b'.code), codePoints("a😀b")) + } + + @Test + fun aLoneSurrogatePassesThroughUnchanged() { + assertEquals(listOf(0xD83D, 'x'.code), codePoints("\uD83Dx")) + } + + @Test + fun navigationKeysMapToDeviceEvents() { + assertEquals(INPUT_UP, RemoteKey.UP.toInputEvent()) + assertEquals(INPUT_DOWN, RemoteKey.DOWN.toInputEvent()) + assertEquals(INPUT_LEFT, RemoteKey.LEFT.toInputEvent()) + assertEquals(INPUT_RIGHT, RemoteKey.RIGHT.toInputEvent()) + assertEquals(INPUT_BACK, RemoteKey.ESCAPE.toInputEvent()) + } + + @Test + fun terminalOnlyKeysMapToNothing() { + assertNull(RemoteKey.HOME.toInputEvent()) + assertNull(RemoteKey.PAGE_DOWN.toInputEvent()) + assertNull(RemoteKey.DELETE.toInputEvent()) + } +} diff --git a/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/MirrorColorResolverTest.kt b/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/MirrorColorResolverTest.kt new file mode 100644 index 00000000000..0561e5da7ef --- /dev/null +++ b/feature/settings/src/commonTest/kotlin/org/meshtastic/feature/settings/debugging/MirrorColorResolverTest.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 Meshtastic LLC + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package org.meshtastic.feature.settings.debugging + +import androidx.compose.ui.graphics.Color +import kotlin.test.Test +import kotlin.test.assertEquals + +class MirrorColorResolverTest { + + @Test + fun `rgb565 expands by bit replication`() { + assertEquals(Color(255, 255, 255), rgb565ToColor(0xFFFF)) + assertEquals(Color(0, 0, 0), rgb565ToColor(0x0000)) + assertEquals(Color(255, 0, 0), rgb565ToColor(0xF800)) + assertEquals(Color(0, 255, 0), rgb565ToColor(0x07E0)) + assertEquals(Color(0, 0, 255), rgb565ToColor(0x001F)) + // 5-bit 16 replicates to 10000100b = 132, where plain scaling floors to 131 + assertEquals(132, (rgb565ToColor(16 shl 11).red * 255).toInt()) + } + + @Test + fun `highest-index region wins where regions overlap`() { + val under = ResolvedRegion(left = 0, top = 0, right = 100, bottom = 10, on = Color.Red, off = Color.Black) + val over = ResolvedRegion(left = 50, top = 0, right = 100, bottom = 10, on = Color.Green, off = Color.Blue) + val rows = listOf(under, over) + + assertEquals( + Color.Red, + resolvePixelColor(x = 10, set = true, rowRegions = rows, defaultOn = Color.White, defaultOff = Color.Black), + ) + assertEquals( + Color.Green, + resolvePixelColor(x = 60, set = true, rowRegions = rows, defaultOn = Color.White, defaultOff = Color.Black), + ) + assertEquals( + Color.Blue, + resolvePixelColor( + x = 60, + set = false, + rowRegions = rows, + defaultOn = Color.White, + defaultOff = Color.Black, + ), + ) + } + + @Test + fun `pixels outside all regions use the defaults`() { + val region = ResolvedRegion(left = 0, top = 0, right = 10, bottom = 10, on = Color.Red, off = Color.Black) + + assertEquals( + Color.White, + resolvePixelColor( + x = 50, + set = true, + rowRegions = listOf(region), + defaultOn = Color.White, + defaultOff = Color.Gray, + ), + ) + assertEquals( + Color.Gray, + resolvePixelColor( + x = 50, + set = false, + rowRegions = listOf(region), + defaultOn = Color.White, + defaultOff = Color.Gray, + ), + ) + } +} diff --git a/gradle.properties b/gradle.properties index d2a21957fe6..59fcb23b693 100644 --- a/gradle.properties +++ b/gradle.properties @@ -40,3 +40,6 @@ org.gradle.priority=low # stays parallel if Isolated Projects is ever off. org.gradle.tooling.parallel=true org.gradle.welcome=never + +# PoC: resolve org.meshtastic:protobufs 2.99.0-screen-mirror-poc-SNAPSHOT from mavenLocal +useMavenLocal=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 0d69db90b53..4ebe4a76887 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -101,6 +101,7 @@ markdownRenderer = "0.46.0" okio = "3.18.2" uri-kmp = "0.0.21" spotless = "8.10.4" +sceneview = "4.56.0" vico = "3.3.1" kable = "0.45.0" mqttastic = "0.9.0" @@ -109,7 +110,10 @@ jmdns = "3.6.4" dbus-java = "5.2.2" qrcode-kotlin = "4.5.0" takpacket-sdk = "0.9.3" -meshtastic-protobufs = "2.8.1.11-g28f4f5f-SNAPSHOT" +# PoC: DisplayFrame screen mirroring, locally published from protobufs@screen-mirror-poc. +# Revert to main's pin once protobufs#1054 lands. main pins 2.8.1.11-g28f4f5f-SNAPSHOT. +# (requires -PuseMavenLocal or the gradle.properties flag on this branch) +meshtastic-protobufs = "2.99.0-screen-mirror-poc-SNAPSHOT" # Gradle Plugins ccud = "2.8.0"