Fetch natives via go run dcc-bigfred/common/cmd/fetch

.github/workflows/ci.yml

@@ -80,21 +80,18 @@ jobs:
           echo "$BIGFRED_STORE_FILE_CONTENT" | base64 -d > "${RUNNER_TEMP}/bigfred-release.jks"
           echo "BIGFRED_STORE_FILE=${RUNNER_TEMP}/bigfred-release.jks" >> "$GITHUB_ENV"
 
-      - name: Set up ORAS
-        uses: oras-project/setup-oras@v1
-
       - name: Fetch native prebuilts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BIGFRED_OCI_TAG: ${{ vars.BIGFRED_OCI_TAG }}
-          MICROINIT_OCI_TAG: ${{ vars.MICROINIT_OCI_TAG }}
+          BIGFRED_REF: ${{ vars.BIGFRED_REF }}
+          MICROINIT_REF: ${{ vars.MICROINIT_REF }}
         run: |
           set -euo pipefail
-          LOCO_TAG="${BIGFRED_OCI_TAG:-main}"
-          MICROINIT_TAG="${MICROINIT_OCI_TAG:-main}"
-          echo "Using loco-server OCI tag: ${LOCO_TAG}"
-          echo "Using microinit OCI tag: ${MICROINIT_TAG}"
-          make native-prebuilt BIGFRED_OCI_TAG="${LOCO_TAG}" MICROINIT_OCI_TAG="${MICROINIT_TAG}"
+          LOCO_REF="${BIGFRED_REF:-master}"
+          MICROINIT_REF="${MICROINIT_REF:-main}"
+          echo "Using loco-server ref: ${LOCO_REF}"
+          echo "Using microinit ref: ${MICROINIT_REF}"
+          make native-prebuilt BIGFRED_REF="${LOCO_REF}" MICROINIT_REF="${MICROINIT_REF}"
           ls -lh native-prebuilt/arm64-v8a/
 
       - name: Resolve app version

.gitignore

@@ -38,6 +38,7 @@ app/src/main/assets/models/
 tools/hydrus-import/out/
 tools/hydrus-import/__pycache__/
 
-# Native prebuilts (GHCR + deps-android-* GitHub Releases) and staged jniLibs
+# Native prebuilts (GitHub tip/Releases) and staged jniLibs
 native-prebuilt/
 app/src/main/jniLibs/
+.ci-github/

Makefile

@@ -5,13 +5,17 @@
 #   make debug   — debug APK
 #   make clean   — remove build outputs
 #   make import-models — fetch hydrus.pl catalog into assets/models/
-#   make loco-android      — download libloco-server.so from GHCR (ORAS)
-#   make valkey-android    — download libvalkey-server.so from deps-android-valkey latest release
-#   make microinit-android — download libmicroinit.so from GHCR (ORAS)
+#   make loco-android      — download libloco-server.so (GitHub tip/Release)
+#   make valkey-android    — download libvalkey-server.so from deps-android-valkey
+#   make microinit-android — download libmicroinit.so (GitHub tip/Release)
+#
+# Native tip/Release fetch: go run github.com/dcc-bigfred/common/cmd/fetch@latest
 
 GRADLE ?= ./gradlew
 GRADLE_FLAGS ?= --quiet
 PYTHON ?= python3
+FETCH_PKG ?= github.com/dcc-bigfred/common/cmd/fetch@latest
+export GOPROXY ?= direct
 
 APK_RELEASE := app/build/outputs/apk/release/app-release.apk
 APK_DEBUG   := app/build/outputs/apk/debug/app-debug.apk
@@ -26,34 +30,22 @@ LOCAL_LOCO_BIN  := ../bigfred/bin/loco-server-android-arm64
 VALKEY_SO       := $(NATIVE_PREBUILT)/libvalkey-server.so
 MICROINIT_SO    := $(NATIVE_PREBUILT)/libmicroinit.so
 
-BIGFRED_OCI_IMAGE ?= ghcr.io/dcc-bigfred/loco-server-android-arm64
-BIGFRED_OCI_TAG   ?= main
-MICROINIT_OCI_IMAGE ?= ghcr.io/dcc-bigfred/microinit-android-arm64
-MICROINIT_OCI_TAG   ?= main
-
-VALKEY_REPO      ?= dcc-bigfred/deps-android-valkey
+BIGFRED_REF    ?= master
+MICROINIT_REF  ?= main
+VALKEY_REPO    ?= dcc-bigfred/deps-android-valkey
 
 .PHONY: help apk release test test-android debug clean import-models \
 	loco-android valkey-android microinit-android native-prebuilt
 
 help:
 	@echo "Targets:"
 	@echo "  make apk                 Build signed release APK → $(APK_RELEASE)"
-	@echo "  make release             Alias for apk"
-	@echo "  make test                Run JVM unit tests"
-	@echo "  make test-android        Run instrumented tests (device/emulator required)"
-	@echo "  make debug               Build debug APK → $(APK_DEBUG)"
-	@echo "  make import-models       Import hydrus models DB + thumbs → $(ASSETS_MODELS)"
-	@echo "  make loco-android        Fetch $(LOCO_SO) (local $(LOCAL_LOCO_BIN) if present, else $(BIGFRED_OCI_IMAGE):$(BIGFRED_OCI_TAG); FORCE=1)"
-	@echo "  make valkey-android      Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release (skip if exists; FORCE=1)"
-	@echo "  make microinit-android   Fetch $(MICROINIT_SO) from $(MICROINIT_OCI_IMAGE):$(MICROINIT_OCI_TAG) (skip if exists; FORCE=1)"
-	@echo "  make clean               Clean Gradle build outputs"
-	@echo ""
-	@echo "Release signing (optional; falls back to debug keystore):"
-	@echo "  BIGFRED_STORE_FILE / BIGFRED_STORE_PASSWORD"
-	@echo "  BIGFRED_KEY_ALIAS  / BIGFRED_KEY_PASSWORD"
+	@echo "  make loco-android        Fetch $(LOCO_SO) (local bin or GitHub $(BIGFRED_REF))"
+	@echo "  make valkey-android      Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release"
+	@echo "  make microinit-android   Fetch $(MICROINIT_SO) from GitHub $(MICROINIT_REF)"
 	@echo ""
-	@echo "Private GitHub deps (optional): GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN"
+	@echo "Tip refs need GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN"
+	@echo "Binary fetch: go run $(FETCH_PKG)"
 
 import-models:
 	$(PYTHON) "$(IMPORT_SCRIPT)" --out "$(IMPORT_OUT)"
@@ -62,11 +54,6 @@ import-models:
 	rm -rf "$(ASSETS_MODELS)/images"
 	cp -a "$(IMPORT_OUT)/images" "$(ASSETS_MODELS)/images"
 	@echo "Assets ready: $(ASSETS_MODELS)"
-	@ls -lh "$(ASSETS_MODELS)/models.db"
-	@echo "Images: $$(find "$(ASSETS_MODELS)/images" -type f | wc -l)"
-
-# --- Native prebuilts (download from deps-* GitHub Releases) -----------------
-# Thin Make rules: skip when the output exists. FORCE=1 removes first.
 
 ifdef FORCE
 .PHONY: force-clean-native
@@ -86,20 +73,50 @@ $(LOCO_SO): $(LOCAL_LOCO_BIN)
 	@echo "Using local $< → $@"
 else
 $(LOCO_SO):
+	@command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; }
 	@mkdir -p "$(NATIVE_PREBUILT)"
-	./scripts/fetch-ghcr-oras.sh "$(BIGFRED_OCI_IMAGE)" "$(BIGFRED_OCI_TAG)" "$@" main
+	@tmpdir="$$(mktemp -d)"; \
+	trap 'rm -rf "$$tmpdir"' EXIT; \
+	go run "$(FETCH_PKG)" \
+		--repo=dcc-bigfred/bigfred \
+		--artifact=binaries \
+		--files=loco-server-android-arm64:bin/libloco-server.so \
+		"$(BIGFRED_REF)" "$$tmpdir/out.tar" && \
+	tar -xOf "$$tmpdir/out.tar" bin/libloco-server.so > "$@"
+	@chmod 755 "$@"
+	@echo "Wrote $@"
 endif
 
 valkey-android: $(VALKEY_SO)
 
 $(VALKEY_SO):
-	./scripts/fetch-github-release-asset.sh "$(VALKEY_REPO)" libvalkey-server.so "$@"
+	@command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; }
+	@mkdir -p "$(NATIVE_PREBUILT)"
+	@tmpdir="$$(mktemp -d)"; \
+	trap 'rm -rf "$$tmpdir"' EXIT; \
+	go run "$(FETCH_PKG)" \
+		--repo="$(VALKEY_REPO)" \
+		--files=libvalkey-server.so:bin/libvalkey-server.so \
+		latest-release "$$tmpdir/out.tar" && \
+	tar -xOf "$$tmpdir/out.tar" bin/libvalkey-server.so > "$@"
+	@chmod 755 "$@"
+	@echo "Wrote $@"
 
 microinit-android: $(MICROINIT_SO)
 
 $(MICROINIT_SO):
+	@command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; }
 	@mkdir -p "$(NATIVE_PREBUILT)"
-	./scripts/fetch-ghcr-oras.sh "$(MICROINIT_OCI_IMAGE)" "$(MICROINIT_OCI_TAG)" "$@" main
+	@tmpdir="$$(mktemp -d)"; \
+	trap 'rm -rf "$$tmpdir"' EXIT; \
+	go run "$(FETCH_PKG)" \
+		--repo=dcc-bigfred/microinit \
+		--artifact=binaries-android-arm64 \
+		--files=libmicroinit.so:bin/libmicroinit.so \
+		"$(MICROINIT_REF)" "$$tmpdir/out.tar" && \
+	tar -xOf "$$tmpdir/out.tar" bin/libmicroinit.so > "$@"
+	@chmod 755 "$@"
+	@echo "Wrote $@"
 
 native-prebuilt: loco-android valkey-android microinit-android
 

app/build.gradle.kts

@@ -150,8 +150,6 @@ val fetchNativeBinaries by tasks.registering {
     val prebuilt = rootProject.layout.projectDirectory.dir("native-prebuilt/arm64-v8a")
     val localLoco = rootProject.layout.projectDirectory.dir("../bigfred/bin")
         .file("loco-server-android-arm64")
-    val fetchScript = rootProject.layout.projectDirectory.file("scripts/fetch-github-release-asset.sh")
-    val ghcrScript = rootProject.layout.projectDirectory.file("scripts/fetch-ghcr-oras.sh")
     outputs.dir(jniOut)
     doLast {
         val outDir = jniOut.asFile
@@ -164,66 +162,58 @@ val fetchNativeBinaries by tasks.registering {
                 ?: System.getenv("GH_TOKEN")
                 ?: System.getenv("GITHUB_TOKEN")
 
-        fun copyIfExists(src: File, destName: String): Boolean {
-            if (!src.isFile) return false
-            src.copyTo(File(outDir, destName), overwrite = true)
-            println("Staged $destName from ${src.absolutePath}")
-            return true
-        }
-
-        fun runScript(script: File, vararg args: String) {
-            require(script.isFile) { "Missing script: ${script.absolutePath}" }
-            val pb = ProcessBuilder(listOf(script.absolutePath, *args))
+        fun runProcess(
+            command: List<String>,
+            label: String,
+            extraEnv: Map<String, String> = emptyMap(),
+        ) {
+            val pb = ProcessBuilder(command)
             pb.directory(rootProject.projectDir)
             pb.redirectErrorStream(true)
             val env = pb.environment()
-            githubToken()?.let { env["GITHUB_TOKEN"] = it }
+            githubToken()?.let {
+                env["GITHUB_TOKEN"] = it
+                env["GH_TOKEN"] = it
+            }
+            env.putAll(extraEnv)
             val proc = pb.start()
             val output = proc.inputStream.bufferedReader().readText()
             val code = proc.waitFor()
             print(output)
             if (code != 0) {
-                throw GradleException("Script ${script.name} failed (exit $code)")
+                throw GradleException("$label failed (exit $code)")
             }
         }
 
-        fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) {
-            runScript(fetchScript.asFile, repo, assetName, dest.absolutePath)
+        fun runMake(vararg targets: String) {
+            runProcess(listOf("make", *targets), "make ${targets.joinToString(" ")}")
         }
 
-        fun fetchLocoFromGhcr(dest: File) {
-            val image = (project.findProperty("bigfredOciImage") as String?)
-                ?.ifBlank { null }
-                ?: System.getenv("BIGFRED_OCI_IMAGE")
-                ?: "ghcr.io/dcc-bigfred/loco-server-android-arm64"
-            val tag = (project.findProperty("bigfredOciTag") as String?)
-                ?.ifBlank { null }
-                ?: System.getenv("BIGFRED_OCI_TAG")
-                ?: "main"
-            runScript(ghcrScript.asFile, image, tag, dest.absolutePath, "main")
+        // Valkey: native-prebuilt → make valkey-android (GitHub latest-release via go fetch).
+        val valkeyDest = File(outDir, "libvalkey-server.so")
+        val valkeyCached = prebuilt.file("libvalkey-server.so").asFile
+        when {
+            valkeyCached.isFile -> {
+                valkeyCached.copyTo(valkeyDest, overwrite = true)
+                println("Staged libvalkey-server.so from ${valkeyCached.absolutePath}")
+            }
+            else -> {
+                runMake("valkey-android")
+                require(valkeyCached.isFile) {
+                    "make valkey-android did not produce ${valkeyCached}"
+                }
+                valkeyCached.copyTo(valkeyDest, overwrite = true)
+            }
         }
-
-        fun stageOrFetch(prebuiltName: String, repoProp: String, defaultRepo: String) {
-            val dest = File(outDir, prebuiltName)
-            if (copyIfExists(prebuilt.file(prebuiltName).asFile, prebuiltName)) return
-            val repo = (project.findProperty(repoProp) as String?)
-                ?.ifBlank { null }
-                ?: defaultRepo
-            fetchLatestReleaseAsset(repo, prebuiltName, dest)
-            // Keep native-prebuilt in sync for make skip-if-exists.
-            val cache = prebuilt.file(prebuiltName).asFile
-            cache.parentFile.mkdirs()
-            dest.copyTo(cache, overwrite = true)
+        if (!valkeyDest.isFile || valkeyDest.length() < 1024) {
+            throw GradleException(
+                "libvalkey-server.so missing after fetch. " +
+                    "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " +
+                    "run 'make valkey-android'.",
+            )
         }
 
-        // Valkey: prefer make-fetched native-prebuilt, else latest GitHub release.
-        stageOrFetch(
-            "libvalkey-server.so",
-            "depsAndroidValkeyRepo",
-            "dcc-bigfred/deps-android-valkey",
-        )
-
-        // loco-server: local ../bigfred/bin → native-prebuilt → GHCR (ORAS).
+        // loco-server: local ../bigfred/bin → native-prebuilt → make loco-android (GitHub).
         val locoDest = File(outDir, "libloco-server.so")
         val locoCached = prebuilt.file("libloco-server.so").asFile
         when {
@@ -236,21 +226,20 @@ val fetchNativeBinaries by tasks.registering {
                 println("Staged libloco-server.so from ${locoCached.absolutePath}")
             }
             else -> {
-                fetchLocoFromGhcr(locoDest)
-                locoDest.parentFile.mkdirs()
-                locoCached.parentFile.mkdirs()
-                locoDest.copyTo(locoCached, overwrite = true)
+                runMake("loco-android")
+                require(locoCached.isFile) { "make loco-android did not produce ${locoCached}" }
+                locoCached.copyTo(locoDest, overwrite = true)
             }
         }
         if (!locoDest.isFile || locoDest.length() < 1024) {
             throw GradleException(
                 "libloco-server.so missing after fetch. " +
                     "Build with 'make -C ../bigfred android', place it in native-prebuilt/arm64-v8a/, " +
-                    "or pull from ghcr.io/dcc-bigfred/loco-server-android-arm64 (ORAS).",
+                    "or set GITHUB_TOKEN and run 'make loco-android'.",
             )
         }
 
-        // microinit: native-prebuilt → GHCR (ORAS).
+        // microinit: native-prebuilt → make microinit-android (GitHub).
         val microinitDest = File(outDir, "libmicroinit.so")
         val microinitCached = prebuilt.file("libmicroinit.so").asFile
         when {
@@ -259,19 +248,18 @@ val fetchNativeBinaries by tasks.registering {
                 println("Staged libmicroinit.so from ${microinitCached.absolutePath}")
             }
             else -> {
-                val image = System.getenv("MICROINIT_OCI_IMAGE")
-                    ?: "ghcr.io/dcc-bigfred/microinit-android-arm64"
-                val tag = System.getenv("MICROINIT_OCI_TAG") ?: "main"
-                runScript(ghcrScript.asFile, image, tag, microinitDest.absolutePath, "main")
-                microinitCached.parentFile.mkdirs()
-                microinitDest.copyTo(microinitCached, overwrite = true)
+                runMake("microinit-android")
+                require(microinitCached.isFile) {
+                    "make microinit-android did not produce ${microinitCached}"
+                }
+                microinitCached.copyTo(microinitDest, overwrite = true)
             }
         }
         if (!microinitDest.isFile || microinitDest.length() < 1024) {
             throw GradleException(
                 "libmicroinit.so missing after fetch. " +
-                    "Place it in native-prebuilt/arm64-v8a/ or pull from " +
-                    "ghcr.io/dcc-bigfred/microinit-android-arm64 (ORAS).",
+                    "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " +
+                    "run 'make microinit-android'.",
             )
         }
     }

gradle.properties

@@ -3,11 +3,7 @@ android.useAndroidX=true
 kotlin.code.style=official
 android.nonTransitiveRClass=true
 
-# Native loco-server for "BigFred on phone" (GHCR OCI via ORAS).
+# Native loco-server / microinit / valkey for "BigFred on phone" (GitHub tip or Releases).
 # Prefer local ../bigfred/bin/loco-server-android-arm64, then native-prebuilt/,
-# then pull from GHCR (:main or release tag via bigfredOciTag).
-bigfredOciImage=ghcr.io/dcc-bigfred/loco-server-android-arm64
-bigfredOciTag=main
-
-# Valkey Android prebuilt (latest GitHub release).
-depsAndroidValkeyRepo=dcc-bigfred/deps-android-valkey
+# then `make loco-android` / `make microinit-android` / `make valkey-android`
+# (see Makefile BIGFRED_REF / MICROINIT_REF / VALKEY_REPO).