diff --git a/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoModels.kt b/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoModels.kt index 059b2e5fc4..c1ed022075 100644 --- a/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoModels.kt +++ b/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoModels.kt @@ -50,6 +50,7 @@ data class OnlineModule( get() = url ?: "https://github.com/Xposed-Modules-Repo/$name" } + data class Collaborator( @SerializedName("login") val login: String?, @SerializedName("name") val name: String?, @@ -268,7 +269,15 @@ data class StoreCatalog( val loaded: Boolean = false, val fromCache: Boolean = false, val loadedAtMillis: Long = 0L, + val page: Int = 1, + val pageCount: Int = 1, + val total: Int = 0, + val isSyncing: Boolean = false, + val syncProgress: Float = 0f, ) { val isEmpty: Boolean get() = modules.isEmpty() + + val isFullySynced: Boolean + get() = page >= pageCount && pageCount > 0 } diff --git a/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoScreen.kt b/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoScreen.kt index 431484882d..4e540aa5d0 100644 --- a/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoScreen.kt +++ b/manager-ui/src/main/kotlin/org/matrix/vector/ui/store/RepoScreen.kt @@ -28,6 +28,7 @@ import androidx.compose.foundation.layout.size import androidx.compose.foundation.layout.width import androidx.compose.foundation.lazy.LazyColumn import androidx.compose.foundation.lazy.items +import androidx.compose.foundation.lazy.rememberLazyListState import androidx.compose.material.icons.Icons import androidx.compose.material.icons.rounded.Check import androidx.compose.material.icons.rounded.CloudOff @@ -41,16 +42,19 @@ import androidx.compose.material3.ExperimentalMaterial3Api import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.LinearProgressIndicator import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold import androidx.compose.material3.Text import androidx.compose.material3.pulltorefresh.PullToRefreshBox import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color @@ -109,7 +113,7 @@ fun RepoScreen( StoreHeader( catalog = catalog, updates = updates, - search = { StoreSearch(query, viewModel, sort, priorities, channel) }, + search = { StoreSearch(query, viewModel, sort, priorities, channel, isSyncing = catalog.isSyncing) }, actions = actions, ) @@ -124,30 +128,65 @@ fun RepoScreen( return@Column } - PullToRefreshBox(isRefreshing = refreshing, onRefresh = viewModel::refresh) { - LazyColumn( - modifier = Modifier.fillMaxSize(), - contentPadding = PaddingValues(bottom = 20.dp), - ) { - if (entries.isEmpty()) { - // Inside the list rather than beside it: an empty Box has nothing to - // scroll, and pull-to-refresh is exactly what the reader wants when the - // reason the list is empty is that the network was down. - item { - EmptyState( - modifier = Modifier.fillParentMaxSize(), - catalog = catalog, - filtered = query.isNotBlank(), - ) - } - } else { - items(entries, key = { it.module.name }) { entry -> - StoreRow(entry = entry, onClick = { onModuleClick(entry.module.name) }) - HorizontalDivider(color = MaterialTheme.colorScheme.surfaceVariant) + Box(modifier = Modifier.weight(1f).fillMaxWidth()) { + PullToRefreshBox(isRefreshing = refreshing, onRefresh = viewModel::refresh) { + LazyColumn( + modifier = Modifier.fillMaxSize(), + contentPadding = PaddingValues(bottom = 20.dp), + ) { + if (entries.isEmpty()) { + // Inside the list rather than beside it: an empty Box has nothing to + // scroll, and pull-to-refresh is exactly what the reader wants when the + // reason the list is empty is that the network was down. + item { + EmptyState( + modifier = Modifier.fillParentMaxSize(), + catalog = catalog, + filtered = query.isNotBlank(), + ) + } + } else { + items(entries, key = { it.module.name }) { entry -> + StoreRow(entry = entry, onClick = { onModuleClick(entry.module.name) }) + HorizontalDivider(color = MaterialTheme.colorScheme.surfaceVariant) + } } } } } + + // Sync progress bar positioned above the bottom bar + if (catalog.isSyncing) { + Column( + modifier = + Modifier.fillMaxWidth() + .padding(horizontal = 20.dp, vertical = 6.dp) + ) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically, + ) { + Text( + text = "Syncing modules (${catalog.page}/${catalog.pageCount})...", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Text( + text = "${(catalog.syncProgress * 100).toInt()}%", + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.primary, + ) + } + Spacer(Modifier.height(4.dp)) + LinearProgressIndicator( + progress = { catalog.syncProgress }, + modifier = Modifier.fillMaxWidth().height(3.dp), + color = MaterialTheme.colorScheme.primary, + trackColor = MaterialTheme.colorScheme.surfaceVariant, + ) + } + } } } } @@ -165,11 +204,12 @@ private fun StoreHeader( actions = actions, description = { if (catalog.modules.isNotEmpty()) { + val count = if (catalog.total > 0) catalog.total else catalog.modules.size val total = context.resources.getQuantityString( R.plurals.store_module_count, - catalog.modules.size, - catalog.modules.size, + count, + count, ) // "Up to date" is stated in words rather than as a zero, because a zero in a row // of counts reads as a failure to load rather than as good news. @@ -200,6 +240,7 @@ private fun StoreSearch( sort: StoreSort, priorities: List, channel: StoreChannel, + isSyncing: Boolean, ) { SearchField( query = query, @@ -213,6 +254,7 @@ private fun StoreSearch( onTogglePriority = viewModel::togglePriority, channel = channel, onChannelChange = viewModel::setChannel, + isSyncing = isSyncing, ) } } @@ -235,6 +277,7 @@ private fun StoreFilterButton( onTogglePriority: (StorePriority) -> Unit, channel: StoreChannel, onChannelChange: (StoreChannel) -> Unit, + isSyncing: Boolean = false, ) { var sheetOpen by remember { mutableStateOf(false) } val narrowed = @@ -242,14 +285,17 @@ private fun StoreFilterButton( channel != StoreChannel.Stable || priorities != listOf(StorePriority.Updates) - IconButton(onClick = { sheetOpen = true }) { + IconButton(onClick = { if (!isSyncing) sheetOpen = true }, enabled = !isSyncing) { BadgedBox(badge = { if (narrowed) Badge(modifier = Modifier.size(6.dp)) }) { Icon( Icons.Rounded.FilterList, contentDescription = stringResource(R.string.store_filter), tint = - if (narrowed) MaterialTheme.colorScheme.primary - else MaterialTheme.colorScheme.onSurfaceVariant, + when { + isSyncing -> MaterialTheme.colorScheme.outlineVariant + narrowed -> MaterialTheme.colorScheme.primary + else -> MaterialTheme.colorScheme.onSurfaceVariant + }, ) } } @@ -427,6 +473,7 @@ private fun StoreRow(entry: StoreEntry, onClick: () -> Unit) { } } + @Composable private fun RowBadge(icon: ImageVector, text: String, tint: Color, modifier: Modifier = Modifier) { Row(modifier = modifier, verticalAlignment = Alignment.CenterVertically) { diff --git a/manager/src/main/kotlin/org/matrix/vector/manager/data/repository/RepoRepository.kt b/manager/src/main/kotlin/org/matrix/vector/manager/data/repository/RepoRepository.kt index b54183dcd0..e977faaec4 100644 --- a/manager/src/main/kotlin/org/matrix/vector/manager/data/repository/RepoRepository.kt +++ b/manager/src/main/kotlin/org/matrix/vector/manager/data/repository/RepoRepository.kt @@ -30,9 +30,7 @@ import org.matrix.vector.manager.logW * The Store's data: the online catalogue, and what this device already has of it. * * **The mirror list is two lists, and that is not an oversight.** The full `modules.json` is served - * by exactly one host today: `modules.lsposed.org` answers that path with a 403. Per-module - * `module/.json` *is* served by both hosts, so the public site is a real fallback there - * and only there. Merging these two lists back into one would quietly take the Store offline. + * Per-module `module/.json` is served by both hosts, with `modules.lsposed.org` providing the latest data and `backup.modules.lsposed.org` serving as fallback. * * **Caching is declared, not hoped for.** Every request states its own freshness, so the 16 MB disk * cache in `HttpClientFactory` is actually used: the catalogue revalidates against the server's own @@ -60,6 +58,7 @@ class RepoRepository( override val isRefreshing: StateFlow = _isRefreshing.asStateFlow() private val _installed = MutableStateFlow>(emptyMap()) + private var lastFullSyncTimeMillis: Long = 0L /** * What each package on this device is at, keyed by package name. @@ -91,12 +90,22 @@ class RepoRepository( * because a user who pulls is telling us they think what they are looking at is stale. */ override suspend fun refresh(force: Boolean) { - // A second caller during a refresh is a no-op rather than a queued duplicate of a 1.2 MB - // download. + // A second caller during a refresh is a no-op rather than a queued duplicate if (!refreshing.tryLock()) return try { _isRefreshing.value = true withContext(Dispatchers.IO) { + // If not forcing a refresh, check if we already have a fully synced catalog within 1 hour + if (!force) { + val current = _catalog.value + val ageMillis = System.currentTimeMillis() - lastFullSyncTimeMillis + val isFresh = lastFullSyncTimeMillis > 0L && ageMillis < TimeUnit.MINUTES.toMillis(CATALOG_MAX_AGE_MINUTES.toLong()) + if (current.isFullySynced && isFresh) { + logI("store: catalog is fresh (age: ${TimeUnit.MILLISECONDS.toMinutes(ageMillis)}m), skipping sync") + return@withContext + } + } + val freshness = if (force) CacheControl.FORCE_NETWORK else @@ -104,34 +113,71 @@ class RepoRepository( .maxAge(CATALOG_MAX_AGE_MINUTES, TimeUnit.MINUTES) .build() - val fetched = LIST_MIRRORS.firstNotNullOfOrNull { fetchCatalog(it, freshness) } - if (fetched != null) { - _catalog.value = fetched + // Fetch page 1 immediately to show the initial modules instantly + val firstPage = fetchPaginatedCatalog(1, freshness) + ?: LIST_MIRRORS.firstNotNullOfOrNull { fetchCatalog(it, freshness) } + + if (firstPage != null) { + val initialCatalog = firstPage.copy( + isSyncing = firstPage.page < firstPage.pageCount, + syncProgress = if (firstPage.pageCount > 0) 1f / firstPage.pageCount else 1f, + ) + _catalog.value = initialCatalog + + // If more pages exist, sync them sequentially in the background + if (firstPage.page < firstPage.pageCount) { + syncRemainingPages(firstPage, freshness) + } else { + lastFullSyncTimeMillis = System.currentTimeMillis() + } return@withContext } - // Every mirror failed. Before reporting nothing, ask the cache — the bytes from - // the last successful visit are usually still on disk, and a stale catalogue is - // far more use than an empty screen. - val cached = - LIST_MIRRORS.firstNotNullOfOrNull { fetchCatalog(it, CacheControl.FORCE_CACHE) } + // If network failed, check cache for page 1 + val cached = fetchPaginatedCatalog(1, CacheControl.FORCE_CACHE) + ?: LIST_MIRRORS.firstNotNullOfOrNull { fetchCatalog(it, CacheControl.FORCE_CACHE) } + when { - cached != null -> _catalog.value = cached.copy(fromCache = true) - // Nothing on the network and nothing on disk. `loaded` still flips, so the - // screen can say the repository is unreachable instead of sitting forever on - // a spinner that means nothing. - else -> _catalog.value = _catalog.value.copy(loaded = true) + cached != null -> _catalog.value = cached.copy(fromCache = true, isSyncing = false) + else -> _catalog.value = _catalog.value.copy(loaded = true, isSyncing = false) } } } finally { - // In a `finally` rather than after the block: a cancelled `viewModelScope` — a - // rotation mid-refresh — would otherwise strand the flag at true, and with - // pull-to-refresh reading it that is a spinner that never stops. _isRefreshing.value = false refreshing.unlock() } } + private suspend fun syncRemainingPages(initial: StoreCatalog, cacheControl: CacheControl) { + var accumulated = initial.modules.toMutableList() + val totalPages = initial.pageCount + for (p in (initial.page + 1)..totalPages) { + val pageResult = fetchPaginatedCatalog(p, cacheControl) + if (pageResult != null) { + accumulated.addAll(pageResult.modules) + val distinct = usable(accumulated) + accumulated = distinct.toMutableList() + val progress = p.toFloat() / totalPages + _catalog.value = _catalog.value.copy( + modules = distinct, + page = p, + pageCount = totalPages, + total = pageResult.total, + isSyncing = p < totalPages, + syncProgress = progress, + ) + } else { + // If a page fails, stop syncing and keep current state + _catalog.value = _catalog.value.copy(isSyncing = false) + break + } + } + _catalog.value = _catalog.value.copy(isSyncing = false, syncProgress = 1f) + if (_catalog.value.isFullySynced) { + lastFullSyncTimeMillis = System.currentTimeMillis() + } + } + /** * The full record for one module: its README, and every release rather than only the newest. * @@ -243,6 +289,60 @@ class RepoRepository( } } + + private fun fetchPaginatedCatalog(page: Int, cacheControl: CacheControl): StoreCatalog? { + val url = "https://modules.lsposed.org/module-list/$page.json" + return try { + client.newCall(request(url, cacheControl)).execute().use { response -> + if (!response.isSuccessful) { + if (response.networkResponse != null) { + logW("store: $url returned HTTP ${response.code}") + } + return null + } + val parsed = parsePaginatedCatalog(response) ?: return null + logI("store: page ${parsed.page}/${parsed.pageCount} (${parsed.modules.size} modules) from $url") + StoreCatalog( + modules = usable(parsed.modules), + loaded = true, + loadedAtMillis = response.receivedResponseAtMillis, + page = parsed.page, + pageCount = parsed.pageCount, + total = parsed.total, + ) + } + } catch (e: Exception) { + logW("store: $url unavailable", e) + null + } + } + + private data class PaginatedResponse( + val modules: List = emptyList(), + val page: Int = 1, + val pageCount: Int = 1, + val total: Int = 0, + ) + + private fun parsePaginatedCatalog(response: Response): PaginatedResponse? { + return try { + val json = JsonParser.parseReader(response.body.charStream()).asJsonObject + val page = json.get("page")?.asInt ?: 1 + val pageCount = json.get("pageCount")?.asInt ?: 1 + val total = json.get("total")?.asInt ?: 0 + val modulesArray = json.getAsJsonArray("modules") ?: return null + val modules = ArrayList(modulesArray.size()) + for (element in modulesArray) { + val module = runCatching { gson.fromJson(element, OnlineModule::class.java) } + if (module.isSuccess) module.getOrNull()?.let(modules::add) + } + PaginatedResponse(modules = modules, page = page, pageCount = pageCount, total = total) + } catch (e: Exception) { + logW("store: failed to parse paginated catalog", e) + null + } + } + private fun request(url: String, cacheControl: CacheControl): Request = Request.Builder().url(url).cacheControl(cacheControl).build() @@ -280,14 +380,13 @@ class RepoRepository( * * `distinctBy` is not superstition about today's data — it is what stops a mirror serving the * same package twice from crashing the Store's `LazyColumn`, which is keyed by package name. - * Entries with no release at all are dropped because there is nothing to install and nothing to - * say about them. + * Entries with no release or latestRelease are dropped. */ private fun usable(parsed: List): List = parsed .asSequence() .filter { it.hide != true } - .filter { !it.releases.isNullOrEmpty() } + .filter { !it.releases.isNullOrEmpty() || !it.latestRelease.isNullOrBlank() } .distinctBy { it.name } .toList() @@ -298,12 +397,12 @@ class RepoRepository( */ val LIST_MIRRORS = listOf("https://backup.modules.lsposed.org/") - /** Detail is served by both hosts, so here the public site is a genuine fallback. */ + /** Detail is served by both hosts; primary site carries the latest data while backup is fallback. */ val DETAIL_MIRRORS = - listOf("https://backup.modules.lsposed.org/", "https://modules.lsposed.org/") + listOf("https://modules.lsposed.org/", "https://backup.modules.lsposed.org/") - /** Matches the server's own `cache-control: max-age=600`, so revalidation stays free. */ - const val CATALOG_MAX_AGE_MINUTES = 10 + /** Matches 1 hour cache duration so full catalogue sync only happens once an hour or on pull-to-refresh. */ + const val CATALOG_MAX_AGE_MINUTES = 60 /** Longer: a module's release history changes far less often than the index does. */ const val DETAIL_MAX_AGE_MINUTES = 60