diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80ff5d8..986e9af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,21 +80,18 @@ jobs: echo "$BIGFRED_STORE_FILE_CONTENT" | base64 -d > "${RUNNER_TEMP}/bigfred-release.jks" echo "BIGFRED_STORE_FILE=${RUNNER_TEMP}/bigfred-release.jks" >> "$GITHUB_ENV" - - name: Set up ORAS - uses: oras-project/setup-oras@v1 - - name: Fetch native prebuilts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BIGFRED_OCI_TAG: ${{ vars.BIGFRED_OCI_TAG }} - MICROINIT_OCI_TAG: ${{ vars.MICROINIT_OCI_TAG }} + BIGFRED_REF: ${{ vars.BIGFRED_REF }} + MICROINIT_REF: ${{ vars.MICROINIT_REF }} run: | set -euo pipefail - LOCO_TAG="${BIGFRED_OCI_TAG:-main}" - MICROINIT_TAG="${MICROINIT_OCI_TAG:-main}" - echo "Using loco-server OCI tag: ${LOCO_TAG}" - echo "Using microinit OCI tag: ${MICROINIT_TAG}" - make native-prebuilt BIGFRED_OCI_TAG="${LOCO_TAG}" MICROINIT_OCI_TAG="${MICROINIT_TAG}" + LOCO_REF="${BIGFRED_REF:-master}" + MICROINIT_REF="${MICROINIT_REF:-main}" + echo "Using loco-server ref: ${LOCO_REF}" + echo "Using microinit ref: ${MICROINIT_REF}" + make native-prebuilt BIGFRED_REF="${LOCO_REF}" MICROINIT_REF="${MICROINIT_REF}" ls -lh native-prebuilt/arm64-v8a/ - name: Resolve app version diff --git a/.gitignore b/.gitignore index c3c85f8..dc0c089 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ app/src/main/assets/models/ tools/hydrus-import/out/ tools/hydrus-import/__pycache__/ -# Native prebuilts (GHCR + deps-android-* GitHub Releases) and staged jniLibs +# Native prebuilts (GitHub tip/Releases) and staged jniLibs native-prebuilt/ app/src/main/jniLibs/ +.ci-github/ diff --git a/Makefile b/Makefile index ae79734..54a5875 100644 --- a/Makefile +++ b/Makefile @@ -5,13 +5,17 @@ # make debug — debug APK # make clean — remove build outputs # make import-models — fetch hydrus.pl catalog into assets/models/ -# make loco-android — download libloco-server.so from GHCR (ORAS) -# make valkey-android — download libvalkey-server.so from deps-android-valkey latest release -# make microinit-android — download libmicroinit.so from GHCR (ORAS) +# make loco-android — download libloco-server.so (GitHub tip/Release) +# make valkey-android — download libvalkey-server.so from deps-android-valkey +# make microinit-android — download libmicroinit.so (GitHub tip/Release) +# +# Native tip/Release fetch: go run github.com/dcc-bigfred/common/cmd/fetch@latest GRADLE ?= ./gradlew GRADLE_FLAGS ?= --quiet PYTHON ?= python3 +FETCH_PKG ?= github.com/dcc-bigfred/common/cmd/fetch@latest +export GOPROXY ?= direct APK_RELEASE := app/build/outputs/apk/release/app-release.apk APK_DEBUG := app/build/outputs/apk/debug/app-debug.apk @@ -26,12 +30,9 @@ LOCAL_LOCO_BIN := ../bigfred/bin/loco-server-android-arm64 VALKEY_SO := $(NATIVE_PREBUILT)/libvalkey-server.so MICROINIT_SO := $(NATIVE_PREBUILT)/libmicroinit.so -BIGFRED_OCI_IMAGE ?= ghcr.io/dcc-bigfred/loco-server-android-arm64 -BIGFRED_OCI_TAG ?= main -MICROINIT_OCI_IMAGE ?= ghcr.io/dcc-bigfred/microinit-android-arm64 -MICROINIT_OCI_TAG ?= main - -VALKEY_REPO ?= dcc-bigfred/deps-android-valkey +BIGFRED_REF ?= master +MICROINIT_REF ?= main +VALKEY_REPO ?= dcc-bigfred/deps-android-valkey .PHONY: help apk release test test-android debug clean import-models \ loco-android valkey-android microinit-android native-prebuilt @@ -39,21 +40,12 @@ VALKEY_REPO ?= dcc-bigfred/deps-android-valkey help: @echo "Targets:" @echo " make apk Build signed release APK → $(APK_RELEASE)" - @echo " make release Alias for apk" - @echo " make test Run JVM unit tests" - @echo " make test-android Run instrumented tests (device/emulator required)" - @echo " make debug Build debug APK → $(APK_DEBUG)" - @echo " make import-models Import hydrus models DB + thumbs → $(ASSETS_MODELS)" - @echo " make loco-android Fetch $(LOCO_SO) (local $(LOCAL_LOCO_BIN) if present, else $(BIGFRED_OCI_IMAGE):$(BIGFRED_OCI_TAG); FORCE=1)" - @echo " make valkey-android Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release (skip if exists; FORCE=1)" - @echo " make microinit-android Fetch $(MICROINIT_SO) from $(MICROINIT_OCI_IMAGE):$(MICROINIT_OCI_TAG) (skip if exists; FORCE=1)" - @echo " make clean Clean Gradle build outputs" - @echo "" - @echo "Release signing (optional; falls back to debug keystore):" - @echo " BIGFRED_STORE_FILE / BIGFRED_STORE_PASSWORD" - @echo " BIGFRED_KEY_ALIAS / BIGFRED_KEY_PASSWORD" + @echo " make loco-android Fetch $(LOCO_SO) (local bin or GitHub $(BIGFRED_REF))" + @echo " make valkey-android Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release" + @echo " make microinit-android Fetch $(MICROINIT_SO) from GitHub $(MICROINIT_REF)" @echo "" - @echo "Private GitHub deps (optional): GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN" + @echo "Tip refs need GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN" + @echo "Binary fetch: go run $(FETCH_PKG)" import-models: $(PYTHON) "$(IMPORT_SCRIPT)" --out "$(IMPORT_OUT)" @@ -62,11 +54,6 @@ import-models: rm -rf "$(ASSETS_MODELS)/images" cp -a "$(IMPORT_OUT)/images" "$(ASSETS_MODELS)/images" @echo "Assets ready: $(ASSETS_MODELS)" - @ls -lh "$(ASSETS_MODELS)/models.db" - @echo "Images: $$(find "$(ASSETS_MODELS)/images" -type f | wc -l)" - -# --- Native prebuilts (download from deps-* GitHub Releases) ----------------- -# Thin Make rules: skip when the output exists. FORCE=1 removes first. ifdef FORCE .PHONY: force-clean-native @@ -86,20 +73,50 @@ $(LOCO_SO): $(LOCAL_LOCO_BIN) @echo "Using local $< → $@" else $(LOCO_SO): + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } @mkdir -p "$(NATIVE_PREBUILT)" - ./scripts/fetch-ghcr-oras.sh "$(BIGFRED_OCI_IMAGE)" "$(BIGFRED_OCI_TAG)" "$@" main + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + go run "$(FETCH_PKG)" \ + --repo=dcc-bigfred/bigfred \ + --artifact=binaries \ + --files=loco-server-android-arm64:bin/libloco-server.so \ + "$(BIGFRED_REF)" "$$tmpdir/out.tar" && \ + tar -xOf "$$tmpdir/out.tar" bin/libloco-server.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" endif valkey-android: $(VALKEY_SO) $(VALKEY_SO): - ./scripts/fetch-github-release-asset.sh "$(VALKEY_REPO)" libvalkey-server.so "$@" + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } + @mkdir -p "$(NATIVE_PREBUILT)" + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + go run "$(FETCH_PKG)" \ + --repo="$(VALKEY_REPO)" \ + --files=libvalkey-server.so:bin/libvalkey-server.so \ + latest-release "$$tmpdir/out.tar" && \ + tar -xOf "$$tmpdir/out.tar" bin/libvalkey-server.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" microinit-android: $(MICROINIT_SO) $(MICROINIT_SO): + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } @mkdir -p "$(NATIVE_PREBUILT)" - ./scripts/fetch-ghcr-oras.sh "$(MICROINIT_OCI_IMAGE)" "$(MICROINIT_OCI_TAG)" "$@" main + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + go run "$(FETCH_PKG)" \ + --repo=dcc-bigfred/microinit \ + --artifact=binaries-android-arm64 \ + --files=libmicroinit.so:bin/libmicroinit.so \ + "$(MICROINIT_REF)" "$$tmpdir/out.tar" && \ + tar -xOf "$$tmpdir/out.tar" bin/libmicroinit.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" native-prebuilt: loco-android valkey-android microinit-android diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 6b4d96f..15fb8c6 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -150,8 +150,6 @@ val fetchNativeBinaries by tasks.registering { val prebuilt = rootProject.layout.projectDirectory.dir("native-prebuilt/arm64-v8a") val localLoco = rootProject.layout.projectDirectory.dir("../bigfred/bin") .file("loco-server-android-arm64") - val fetchScript = rootProject.layout.projectDirectory.file("scripts/fetch-github-release-asset.sh") - val ghcrScript = rootProject.layout.projectDirectory.file("scripts/fetch-ghcr-oras.sh") outputs.dir(jniOut) doLast { val outDir = jniOut.asFile @@ -164,66 +162,58 @@ val fetchNativeBinaries by tasks.registering { ?: System.getenv("GH_TOKEN") ?: System.getenv("GITHUB_TOKEN") - fun copyIfExists(src: File, destName: String): Boolean { - if (!src.isFile) return false - src.copyTo(File(outDir, destName), overwrite = true) - println("Staged $destName from ${src.absolutePath}") - return true - } - - fun runScript(script: File, vararg args: String) { - require(script.isFile) { "Missing script: ${script.absolutePath}" } - val pb = ProcessBuilder(listOf(script.absolutePath, *args)) + fun runProcess( + command: List, + label: String, + extraEnv: Map = emptyMap(), + ) { + val pb = ProcessBuilder(command) pb.directory(rootProject.projectDir) pb.redirectErrorStream(true) val env = pb.environment() - githubToken()?.let { env["GITHUB_TOKEN"] = it } + githubToken()?.let { + env["GITHUB_TOKEN"] = it + env["GH_TOKEN"] = it + } + env.putAll(extraEnv) val proc = pb.start() val output = proc.inputStream.bufferedReader().readText() val code = proc.waitFor() print(output) if (code != 0) { - throw GradleException("Script ${script.name} failed (exit $code)") + throw GradleException("$label failed (exit $code)") } } - fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) { - runScript(fetchScript.asFile, repo, assetName, dest.absolutePath) + fun runMake(vararg targets: String) { + runProcess(listOf("make", *targets), "make ${targets.joinToString(" ")}") } - fun fetchLocoFromGhcr(dest: File) { - val image = (project.findProperty("bigfredOciImage") as String?) - ?.ifBlank { null } - ?: System.getenv("BIGFRED_OCI_IMAGE") - ?: "ghcr.io/dcc-bigfred/loco-server-android-arm64" - val tag = (project.findProperty("bigfredOciTag") as String?) - ?.ifBlank { null } - ?: System.getenv("BIGFRED_OCI_TAG") - ?: "main" - runScript(ghcrScript.asFile, image, tag, dest.absolutePath, "main") + // Valkey: native-prebuilt → make valkey-android (GitHub latest-release via go fetch). + val valkeyDest = File(outDir, "libvalkey-server.so") + val valkeyCached = prebuilt.file("libvalkey-server.so").asFile + when { + valkeyCached.isFile -> { + valkeyCached.copyTo(valkeyDest, overwrite = true) + println("Staged libvalkey-server.so from ${valkeyCached.absolutePath}") + } + else -> { + runMake("valkey-android") + require(valkeyCached.isFile) { + "make valkey-android did not produce ${valkeyCached}" + } + valkeyCached.copyTo(valkeyDest, overwrite = true) + } } - - fun stageOrFetch(prebuiltName: String, repoProp: String, defaultRepo: String) { - val dest = File(outDir, prebuiltName) - if (copyIfExists(prebuilt.file(prebuiltName).asFile, prebuiltName)) return - val repo = (project.findProperty(repoProp) as String?) - ?.ifBlank { null } - ?: defaultRepo - fetchLatestReleaseAsset(repo, prebuiltName, dest) - // Keep native-prebuilt in sync for make skip-if-exists. - val cache = prebuilt.file(prebuiltName).asFile - cache.parentFile.mkdirs() - dest.copyTo(cache, overwrite = true) + if (!valkeyDest.isFile || valkeyDest.length() < 1024) { + throw GradleException( + "libvalkey-server.so missing after fetch. " + + "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " + + "run 'make valkey-android'.", + ) } - // Valkey: prefer make-fetched native-prebuilt, else latest GitHub release. - stageOrFetch( - "libvalkey-server.so", - "depsAndroidValkeyRepo", - "dcc-bigfred/deps-android-valkey", - ) - - // loco-server: local ../bigfred/bin → native-prebuilt → GHCR (ORAS). + // loco-server: local ../bigfred/bin → native-prebuilt → make loco-android (GitHub). val locoDest = File(outDir, "libloco-server.so") val locoCached = prebuilt.file("libloco-server.so").asFile when { @@ -236,21 +226,20 @@ val fetchNativeBinaries by tasks.registering { println("Staged libloco-server.so from ${locoCached.absolutePath}") } else -> { - fetchLocoFromGhcr(locoDest) - locoDest.parentFile.mkdirs() - locoCached.parentFile.mkdirs() - locoDest.copyTo(locoCached, overwrite = true) + runMake("loco-android") + require(locoCached.isFile) { "make loco-android did not produce ${locoCached}" } + locoCached.copyTo(locoDest, overwrite = true) } } if (!locoDest.isFile || locoDest.length() < 1024) { throw GradleException( "libloco-server.so missing after fetch. " + "Build with 'make -C ../bigfred android', place it in native-prebuilt/arm64-v8a/, " + - "or pull from ghcr.io/dcc-bigfred/loco-server-android-arm64 (ORAS).", + "or set GITHUB_TOKEN and run 'make loco-android'.", ) } - // microinit: native-prebuilt → GHCR (ORAS). + // microinit: native-prebuilt → make microinit-android (GitHub). val microinitDest = File(outDir, "libmicroinit.so") val microinitCached = prebuilt.file("libmicroinit.so").asFile when { @@ -259,19 +248,18 @@ val fetchNativeBinaries by tasks.registering { println("Staged libmicroinit.so from ${microinitCached.absolutePath}") } else -> { - val image = System.getenv("MICROINIT_OCI_IMAGE") - ?: "ghcr.io/dcc-bigfred/microinit-android-arm64" - val tag = System.getenv("MICROINIT_OCI_TAG") ?: "main" - runScript(ghcrScript.asFile, image, tag, microinitDest.absolutePath, "main") - microinitCached.parentFile.mkdirs() - microinitDest.copyTo(microinitCached, overwrite = true) + runMake("microinit-android") + require(microinitCached.isFile) { + "make microinit-android did not produce ${microinitCached}" + } + microinitCached.copyTo(microinitDest, overwrite = true) } } if (!microinitDest.isFile || microinitDest.length() < 1024) { throw GradleException( "libmicroinit.so missing after fetch. " + - "Place it in native-prebuilt/arm64-v8a/ or pull from " + - "ghcr.io/dcc-bigfred/microinit-android-arm64 (ORAS).", + "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " + + "run 'make microinit-android'.", ) } } diff --git a/gradle.properties b/gradle.properties index ce97ec0..395f1fb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,11 +3,7 @@ android.useAndroidX=true kotlin.code.style=official android.nonTransitiveRClass=true -# Native loco-server for "BigFred on phone" (GHCR OCI via ORAS). +# Native loco-server / microinit / valkey for "BigFred on phone" (GitHub tip or Releases). # Prefer local ../bigfred/bin/loco-server-android-arm64, then native-prebuilt/, -# then pull from GHCR (:main or release tag via bigfredOciTag). -bigfredOciImage=ghcr.io/dcc-bigfred/loco-server-android-arm64 -bigfredOciTag=main - -# Valkey Android prebuilt (latest GitHub release). -depsAndroidValkeyRepo=dcc-bigfred/deps-android-valkey +# then `make loco-android` / `make microinit-android` / `make valkey-android` +# (see Makefile BIGFRED_REF / MICROINIT_REF / VALKEY_REPO). diff --git a/scripts/fetch-ghcr-oras.sh b/scripts/fetch-ghcr-oras.sh deleted file mode 100755 index 17874d5..0000000 --- a/scripts/fetch-ghcr-oras.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Pull a single-file native binary artifact from GHCR (ORAS). -# Usage: fetch-ghcr-oras.sh [fallback-tag ...] -# -# Auth (optional; needed for private packages): -# GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN -set -euo pipefail - -IMAGE="${1:?usage: $0 [fallback-tag ...]}" -TAG="${2:?}" -OUT="${3:?}" -shift 3 -FALLBACK_TAGS=("$@") - -TOKEN="${BIGFRED_NATIVE_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}" -if [[ -n "${TOKEN}" ]]; then - USER="${GITHUB_ACTOR:-oauth2}" - echo "${TOKEN}" | oras login ghcr.io -u "${USER}" --password-stdin >/dev/null -fi - -tmpdir="$(mktemp -d)" -cleanup() { rm -rf "${tmpdir}"; } -trap cleanup EXIT - -pull_tag() { - local t="$1" - rm -rf "${tmpdir:?}"/* - mkdir -p "${tmpdir}" - echo "Pulling ${IMAGE}:${t}…" - oras pull "${IMAGE}:${t}" -o "${tmpdir}" -} - -if ! pull_tag "${TAG}"; then - pulled=0 - for fb in "${FALLBACK_TAGS[@]}"; do - if pull_tag "${fb}"; then - pulled=1 - break - fi - done - if [[ "${pulled}" -eq 0 ]]; then - echo "error: could not pull ${IMAGE}:${TAG} (tried fallbacks: ${FALLBACK_TAGS[*]:-none})" >&2 - exit 1 - fi -fi - -# Prefer a file matching the requested lib*.so output (for example, -# libmicroinit.so → microinit-android-arm64), then retain the historical -# loco-server name and finally accept a single-file artifact. -output_name="$(basename "${OUT}")" -binary_name="${output_name#lib}" -binary_name="${binary_name%.so}" -src="${tmpdir}/${output_name}" -if [[ ! -f "${src}" ]]; then - src="${tmpdir}/${binary_name}-android-arm64" -fi -if [[ ! -f "${src}" ]]; then - src="${tmpdir}/loco-server-android-arm64" - if [[ ! -f "${src}" ]]; then - mapfile -t files < <(find "${tmpdir}" -type f ! -name 'manifest.json' ! -name 'config.json') - if [[ ${#files[@]} -eq 1 ]]; then - src="${files[0]}" - else - echo "error: could not identify executable for ${output_name}; found:" >&2 - find "${tmpdir}" -type f >&2 - exit 1 - fi - fi -fi - -mkdir -p "$(dirname "${OUT}")" -cp -f "${src}" "${OUT}" -chmod 755 "${OUT}" -echo "Wrote ${OUT} ($(wc -c < "${OUT}") bytes) from ${IMAGE}:${TAG}" diff --git a/scripts/fetch-github-release-asset.sh b/scripts/fetch-github-release-asset.sh deleted file mode 100755 index 4d817a2..0000000 --- a/scripts/fetch-github-release-asset.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Download a single asset from the latest GitHub release of a repo. -# Usage: ./scripts/fetch-github-release-asset.sh -# -# Auth (optional, needed for private repos): -# GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN -set -euo pipefail - -REPO="${1:?usage: $0 }" -ASSET="${2:?}" -OUT="${3:?}" - -mkdir -p "$(dirname "${OUT}")" - -TOKEN="${BIGFRED_NATIVE_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}" -AUTH=() -if [[ -n "${TOKEN}" ]]; then - AUTH=(-H "Authorization: Bearer ${TOKEN}") -fi - -API="https://api.github.com/repos/${REPO}/releases/latest" -echo "Resolving latest release of ${REPO}…" -json="$(curl -fsSL "${AUTH[@]}" -H "Accept: application/vnd.github+json" "${API}")" - -tag="$(printf '%s' "${json}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("tag_name",""))')" -url="$(printf '%s' "${json}" | python3 -c ' -import json,sys -asset=sys.argv[1] -data=json.load(sys.stdin) -for a in data.get("assets") or []: - if a.get("name")==asset: - print(a.get("url") or "") - break -' "${ASSET}")" - -if [[ -z "${url}" ]]; then - echo "error: asset '${ASSET}' not found in latest release of ${REPO} (tag=${tag:-unknown})" >&2 - echo "Available assets:" >&2 - printf '%s' "${json}" | python3 -c 'import json,sys; [print(" -",a.get("name")) for a in (json.load(sys.stdin).get("assets") or [])]' >&2 - exit 1 -fi - -echo "Downloading ${ASSET} from ${REPO}@${tag}" -tmp="$(mktemp)" -cleanup() { rm -f "${tmp}"; } -trap cleanup EXIT - -# Asset API URL requires Accept: application/octet-stream -curl -fsSL "${AUTH[@]}" \ - -H "Accept: application/octet-stream" \ - -L \ - -o "${tmp}" \ - "${url}" - -cp -f "${tmp}" "${OUT}" -chmod 755 "${OUT}" -echo "Wrote ${OUT} ($(wc -c < "${OUT}") bytes) from ${REPO}@${tag}"