From 482b2115f4ecef094a8cfb8864d0dde59cdec7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= <372403+keskad@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:39:04 +0200 Subject: [PATCH 1/5] Fetch loco-server and microinit natives via shared GitHub fetch. Drop ORAS/GHCR pull; Makefile clones dcc-bigfred/.github @ v2 and uses curl-based Actions/Release download (no gh CLI). Co-authored-by: Cursor --- .github/workflows/ci.yml | 17 ++++---- .gitignore | 3 +- Makefile | 79 +++++++++++++++++++++++--------------- app/build.gradle.kts | 60 +++++++++++++++-------------- gradle.properties | 6 +-- scripts/fetch-ghcr-oras.sh | 74 ----------------------------------- 6 files changed, 90 insertions(+), 149 deletions(-) delete mode 100755 scripts/fetch-ghcr-oras.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80ff5d8..d207a7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -80,21 +80,18 @@ jobs: echo "$BIGFRED_STORE_FILE_CONTENT" | base64 -d > "${RUNNER_TEMP}/bigfred-release.jks" echo "BIGFRED_STORE_FILE=${RUNNER_TEMP}/bigfred-release.jks" >> "$GITHUB_ENV" - - name: Set up ORAS - uses: oras-project/setup-oras@v1 - - name: Fetch native prebuilts env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BIGFRED_OCI_TAG: ${{ vars.BIGFRED_OCI_TAG }} - MICROINIT_OCI_TAG: ${{ vars.MICROINIT_OCI_TAG }} + BIGFRED_REF: ${{ vars.BIGFRED_REF }} + MICROINIT_REF: ${{ vars.MICROINIT_REF }} run: | set -euo pipefail - LOCO_TAG="${BIGFRED_OCI_TAG:-main}" - MICROINIT_TAG="${MICROINIT_OCI_TAG:-main}" - echo "Using loco-server OCI tag: ${LOCO_TAG}" - echo "Using microinit OCI tag: ${MICROINIT_TAG}" - make native-prebuilt BIGFRED_OCI_TAG="${LOCO_TAG}" MICROINIT_OCI_TAG="${MICROINIT_TAG}" + LOCO_REF="${BIGFRED_REF:-main}" + MICROINIT_REF="${MICROINIT_REF:-main}" + echo "Using loco-server ref: ${LOCO_REF}" + echo "Using microinit ref: ${MICROINIT_REF}" + make native-prebuilt BIGFRED_REF="${LOCO_REF}" MICROINIT_REF="${MICROINIT_REF}" ls -lh native-prebuilt/arm64-v8a/ - name: Resolve app version diff --git a/.gitignore b/.gitignore index c3c85f8..dc0c089 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ app/src/main/assets/models/ tools/hydrus-import/out/ tools/hydrus-import/__pycache__/ -# Native prebuilts (GHCR + deps-android-* GitHub Releases) and staged jniLibs +# Native prebuilts (GitHub tip/Releases) and staged jniLibs native-prebuilt/ app/src/main/jniLibs/ +.ci-github/ diff --git a/Makefile b/Makefile index ae79734..57341e5 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,9 @@ # make debug — debug APK # make clean — remove build outputs # make import-models — fetch hydrus.pl catalog into assets/models/ -# make loco-android — download libloco-server.so from GHCR (ORAS) -# make valkey-android — download libvalkey-server.so from deps-android-valkey latest release -# make microinit-android — download libmicroinit.so from GHCR (ORAS) +# make loco-android — download libloco-server.so (GitHub tip/Release) +# make valkey-android — download libvalkey-server.so from deps-android-valkey +# make microinit-android — download libmicroinit.so (GitHub tip/Release) GRADLE ?= ./gradlew GRADLE_FLAGS ?= --quiet @@ -26,34 +26,27 @@ LOCAL_LOCO_BIN := ../bigfred/bin/loco-server-android-arm64 VALKEY_SO := $(NATIVE_PREBUILT)/libvalkey-server.so MICROINIT_SO := $(NATIVE_PREBUILT)/libmicroinit.so -BIGFRED_OCI_IMAGE ?= ghcr.io/dcc-bigfred/loco-server-android-arm64 -BIGFRED_OCI_TAG ?= main -MICROINIT_OCI_IMAGE ?= ghcr.io/dcc-bigfred/microinit-android-arm64 -MICROINIT_OCI_TAG ?= main +BIGFRED_REF ?= main +MICROINIT_REF ?= main +VALKEY_REPO ?= dcc-bigfred/deps-android-valkey -VALKEY_REPO ?= dcc-bigfred/deps-android-valkey +CI_SCRIPTS_REPO ?= https://github.com/dcc-bigfred/.github.git +CI_SCRIPTS_REF ?= v2 +CI_SCRIPTS_DIR ?= .ci-github .PHONY: help apk release test test-android debug clean import-models \ - loco-android valkey-android microinit-android native-prebuilt + loco-android valkey-android microinit-android native-prebuilt \ + ci-scripts ci-scripts-update help: @echo "Targets:" @echo " make apk Build signed release APK → $(APK_RELEASE)" - @echo " make release Alias for apk" - @echo " make test Run JVM unit tests" - @echo " make test-android Run instrumented tests (device/emulator required)" - @echo " make debug Build debug APK → $(APK_DEBUG)" - @echo " make import-models Import hydrus models DB + thumbs → $(ASSETS_MODELS)" - @echo " make loco-android Fetch $(LOCO_SO) (local $(LOCAL_LOCO_BIN) if present, else $(BIGFRED_OCI_IMAGE):$(BIGFRED_OCI_TAG); FORCE=1)" - @echo " make valkey-android Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release (skip if exists; FORCE=1)" - @echo " make microinit-android Fetch $(MICROINIT_SO) from $(MICROINIT_OCI_IMAGE):$(MICROINIT_OCI_TAG) (skip if exists; FORCE=1)" - @echo " make clean Clean Gradle build outputs" + @echo " make loco-android Fetch $(LOCO_SO) (local bin or GitHub $(BIGFRED_REF))" + @echo " make valkey-android Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release" + @echo " make microinit-android Fetch $(MICROINIT_SO) from GitHub $(MICROINIT_REF)" + @echo " make ci-scripts Clone dcc-bigfred/.github @ $(CI_SCRIPTS_REF)" @echo "" - @echo "Release signing (optional; falls back to debug keystore):" - @echo " BIGFRED_STORE_FILE / BIGFRED_STORE_PASSWORD" - @echo " BIGFRED_KEY_ALIAS / BIGFRED_KEY_PASSWORD" - @echo "" - @echo "Private GitHub deps (optional): GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN" + @echo "Tip refs need GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN" import-models: $(PYTHON) "$(IMPORT_SCRIPT)" --out "$(IMPORT_OUT)" @@ -62,11 +55,19 @@ import-models: rm -rf "$(ASSETS_MODELS)/images" cp -a "$(IMPORT_OUT)/images" "$(ASSETS_MODELS)/images" @echo "Assets ready: $(ASSETS_MODELS)" - @ls -lh "$(ASSETS_MODELS)/models.db" - @echo "Images: $$(find "$(ASSETS_MODELS)/images" -type f | wc -l)" -# --- Native prebuilts (download from deps-* GitHub Releases) ----------------- -# Thin Make rules: skip when the output exists. FORCE=1 removes first. +$(CI_SCRIPTS_DIR)/.ok: + @echo "Cloning $(CI_SCRIPTS_REPO) @ $(CI_SCRIPTS_REF) → $(CI_SCRIPTS_DIR)" + @rm -rf "$(CI_SCRIPTS_DIR)" + @git clone --depth 1 --branch "$(CI_SCRIPTS_REF)" "$(CI_SCRIPTS_REPO)" "$(CI_SCRIPTS_DIR)" \ + || { echo "error: failed to clone $(CI_SCRIPTS_REPO) @ $(CI_SCRIPTS_REF)"; exit 1; } + @touch "$@" + +ci-scripts: $(CI_SCRIPTS_DIR)/.ok + +ci-scripts-update: + rm -rf "$(CI_SCRIPTS_DIR)" + $(MAKE) "$(CI_SCRIPTS_DIR)/.ok" ifdef FORCE .PHONY: force-clean-native @@ -85,9 +86,17 @@ $(LOCO_SO): $(LOCAL_LOCO_BIN) @cp "$<" "$@" @echo "Using local $< → $@" else -$(LOCO_SO): +$(LOCO_SO): $(CI_SCRIPTS_DIR)/.ok @mkdir -p "$(NATIVE_PREBUILT)" - ./scripts/fetch-ghcr-oras.sh "$(BIGFRED_OCI_IMAGE)" "$(BIGFRED_OCI_TAG)" "$@" main + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + GITHUB_REPO=dcc-bigfred/bigfred \ + ARTIFACT_NAME=binaries \ + FILES=loco-server-android-arm64:bin/libloco-server.so \ + "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(BIGFRED_REF)" "$$tmpdir/out.tar"; \ + tar -xOf "$$tmpdir/out.tar" bin/libloco-server.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" endif valkey-android: $(VALKEY_SO) @@ -97,9 +106,17 @@ $(VALKEY_SO): microinit-android: $(MICROINIT_SO) -$(MICROINIT_SO): +$(MICROINIT_SO): $(CI_SCRIPTS_DIR)/.ok @mkdir -p "$(NATIVE_PREBUILT)" - ./scripts/fetch-ghcr-oras.sh "$(MICROINIT_OCI_IMAGE)" "$(MICROINIT_OCI_TAG)" "$@" main + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + GITHUB_REPO=dcc-bigfred/microinit \ + ARTIFACT_NAME=binaries-android-arm64 \ + FILES=libmicroinit.so:bin/libmicroinit.so \ + "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(MICROINIT_REF)" "$$tmpdir/out.tar"; \ + tar -xOf "$$tmpdir/out.tar" bin/libmicroinit.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" native-prebuilt: loco-android valkey-android microinit-android diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 6b4d96f..2f7e2b1 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -151,7 +151,6 @@ val fetchNativeBinaries by tasks.registering { val localLoco = rootProject.layout.projectDirectory.dir("../bigfred/bin") .file("loco-server-android-arm64") val fetchScript = rootProject.layout.projectDirectory.file("scripts/fetch-github-release-asset.sh") - val ghcrScript = rootProject.layout.projectDirectory.file("scripts/fetch-ghcr-oras.sh") outputs.dir(jniOut) doLast { val outDir = jniOut.asFile @@ -187,20 +186,26 @@ val fetchNativeBinaries by tasks.registering { } } - fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) { - runScript(fetchScript.asFile, repo, assetName, dest.absolutePath) + fun runMake(vararg targets: String) { + val pb = ProcessBuilder(listOf("make", *targets)) + pb.directory(rootProject.projectDir) + pb.redirectErrorStream(true) + val env = pb.environment() + githubToken()?.let { + env["GITHUB_TOKEN"] = it + env["GH_TOKEN"] = it + } + val proc = pb.start() + val output = proc.inputStream.bufferedReader().readText() + val code = proc.waitFor() + print(output) + if (code != 0) { + throw GradleException("make ${targets.joinToString(" ")} failed (exit $code)") + } } - fun fetchLocoFromGhcr(dest: File) { - val image = (project.findProperty("bigfredOciImage") as String?) - ?.ifBlank { null } - ?: System.getenv("BIGFRED_OCI_IMAGE") - ?: "ghcr.io/dcc-bigfred/loco-server-android-arm64" - val tag = (project.findProperty("bigfredOciTag") as String?) - ?.ifBlank { null } - ?: System.getenv("BIGFRED_OCI_TAG") - ?: "main" - runScript(ghcrScript.asFile, image, tag, dest.absolutePath, "main") + fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) { + runScript(fetchScript.asFile, repo, assetName, dest.absolutePath) } fun stageOrFetch(prebuiltName: String, repoProp: String, defaultRepo: String) { @@ -210,7 +215,6 @@ val fetchNativeBinaries by tasks.registering { ?.ifBlank { null } ?: defaultRepo fetchLatestReleaseAsset(repo, prebuiltName, dest) - // Keep native-prebuilt in sync for make skip-if-exists. val cache = prebuilt.file(prebuiltName).asFile cache.parentFile.mkdirs() dest.copyTo(cache, overwrite = true) @@ -223,7 +227,7 @@ val fetchNativeBinaries by tasks.registering { "dcc-bigfred/deps-android-valkey", ) - // loco-server: local ../bigfred/bin → native-prebuilt → GHCR (ORAS). + // loco-server: local ../bigfred/bin → native-prebuilt → make loco-android (GitHub). val locoDest = File(outDir, "libloco-server.so") val locoCached = prebuilt.file("libloco-server.so").asFile when { @@ -236,21 +240,20 @@ val fetchNativeBinaries by tasks.registering { println("Staged libloco-server.so from ${locoCached.absolutePath}") } else -> { - fetchLocoFromGhcr(locoDest) - locoDest.parentFile.mkdirs() - locoCached.parentFile.mkdirs() - locoDest.copyTo(locoCached, overwrite = true) + runMake("loco-android") + require(locoCached.isFile) { "make loco-android did not produce ${locoCached}" } + locoCached.copyTo(locoDest, overwrite = true) } } if (!locoDest.isFile || locoDest.length() < 1024) { throw GradleException( "libloco-server.so missing after fetch. " + "Build with 'make -C ../bigfred android', place it in native-prebuilt/arm64-v8a/, " + - "or pull from ghcr.io/dcc-bigfred/loco-server-android-arm64 (ORAS).", + "or set GITHUB_TOKEN and run 'make loco-android'.", ) } - // microinit: native-prebuilt → GHCR (ORAS). + // microinit: native-prebuilt → make microinit-android (GitHub). val microinitDest = File(outDir, "libmicroinit.so") val microinitCached = prebuilt.file("libmicroinit.so").asFile when { @@ -259,19 +262,18 @@ val fetchNativeBinaries by tasks.registering { println("Staged libmicroinit.so from ${microinitCached.absolutePath}") } else -> { - val image = System.getenv("MICROINIT_OCI_IMAGE") - ?: "ghcr.io/dcc-bigfred/microinit-android-arm64" - val tag = System.getenv("MICROINIT_OCI_TAG") ?: "main" - runScript(ghcrScript.asFile, image, tag, microinitDest.absolutePath, "main") - microinitCached.parentFile.mkdirs() - microinitDest.copyTo(microinitCached, overwrite = true) + runMake("microinit-android") + require(microinitCached.isFile) { + "make microinit-android did not produce ${microinitCached}" + } + microinitCached.copyTo(microinitDest, overwrite = true) } } if (!microinitDest.isFile || microinitDest.length() < 1024) { throw GradleException( "libmicroinit.so missing after fetch. " + - "Place it in native-prebuilt/arm64-v8a/ or pull from " + - "ghcr.io/dcc-bigfred/microinit-android-arm64 (ORAS).", + "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " + + "run 'make microinit-android'.", ) } } diff --git a/gradle.properties b/gradle.properties index ce97ec0..4bf84eb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,11 +3,9 @@ android.useAndroidX=true kotlin.code.style=official android.nonTransitiveRClass=true -# Native loco-server for "BigFred on phone" (GHCR OCI via ORAS). +# Native loco-server / microinit for "BigFred on phone" (GitHub Actions tip or Releases). # Prefer local ../bigfred/bin/loco-server-android-arm64, then native-prebuilt/, -# then pull from GHCR (:main or release tag via bigfredOciTag). -bigfredOciImage=ghcr.io/dcc-bigfred/loco-server-android-arm64 -bigfredOciTag=main +# then `make loco-android` / `make microinit-android` (see Makefile BIGFRED_REF / MICROINIT_REF). # Valkey Android prebuilt (latest GitHub release). depsAndroidValkeyRepo=dcc-bigfred/deps-android-valkey diff --git a/scripts/fetch-ghcr-oras.sh b/scripts/fetch-ghcr-oras.sh deleted file mode 100755 index 17874d5..0000000 --- a/scripts/fetch-ghcr-oras.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash -# Pull a single-file native binary artifact from GHCR (ORAS). -# Usage: fetch-ghcr-oras.sh [fallback-tag ...] -# -# Auth (optional; needed for private packages): -# GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN -set -euo pipefail - -IMAGE="${1:?usage: $0 [fallback-tag ...]}" -TAG="${2:?}" -OUT="${3:?}" -shift 3 -FALLBACK_TAGS=("$@") - -TOKEN="${BIGFRED_NATIVE_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}" -if [[ -n "${TOKEN}" ]]; then - USER="${GITHUB_ACTOR:-oauth2}" - echo "${TOKEN}" | oras login ghcr.io -u "${USER}" --password-stdin >/dev/null -fi - -tmpdir="$(mktemp -d)" -cleanup() { rm -rf "${tmpdir}"; } -trap cleanup EXIT - -pull_tag() { - local t="$1" - rm -rf "${tmpdir:?}"/* - mkdir -p "${tmpdir}" - echo "Pulling ${IMAGE}:${t}…" - oras pull "${IMAGE}:${t}" -o "${tmpdir}" -} - -if ! pull_tag "${TAG}"; then - pulled=0 - for fb in "${FALLBACK_TAGS[@]}"; do - if pull_tag "${fb}"; then - pulled=1 - break - fi - done - if [[ "${pulled}" -eq 0 ]]; then - echo "error: could not pull ${IMAGE}:${TAG} (tried fallbacks: ${FALLBACK_TAGS[*]:-none})" >&2 - exit 1 - fi -fi - -# Prefer a file matching the requested lib*.so output (for example, -# libmicroinit.so → microinit-android-arm64), then retain the historical -# loco-server name and finally accept a single-file artifact. -output_name="$(basename "${OUT}")" -binary_name="${output_name#lib}" -binary_name="${binary_name%.so}" -src="${tmpdir}/${output_name}" -if [[ ! -f "${src}" ]]; then - src="${tmpdir}/${binary_name}-android-arm64" -fi -if [[ ! -f "${src}" ]]; then - src="${tmpdir}/loco-server-android-arm64" - if [[ ! -f "${src}" ]]; then - mapfile -t files < <(find "${tmpdir}" -type f ! -name 'manifest.json' ! -name 'config.json') - if [[ ${#files[@]} -eq 1 ]]; then - src="${files[0]}" - else - echo "error: could not identify executable for ${output_name}; found:" >&2 - find "${tmpdir}" -type f >&2 - exit 1 - fi - fi -fi - -mkdir -p "$(dirname "${OUT}")" -cp -f "${src}" "${OUT}" -chmod 755 "${OUT}" -echo "Wrote ${OUT} ($(wc -c < "${OUT}") bytes) from ${IMAGE}:${TAG}" From 9e59159d2d890806b98731ae053e57d0b3c1a9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= <372403+keskad@users.noreply.github.com> Date: Thu, 6 Aug 2026 15:42:10 +0200 Subject: [PATCH 2/5] Default loco-server tip ref to master; fail fetch before tar extract. Co-authored-by: Cursor --- .github/workflows/ci.yml | 2 +- Makefile | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d207a7a..986e9af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,7 +87,7 @@ jobs: MICROINIT_REF: ${{ vars.MICROINIT_REF }} run: | set -euo pipefail - LOCO_REF="${BIGFRED_REF:-main}" + LOCO_REF="${BIGFRED_REF:-master}" MICROINIT_REF="${MICROINIT_REF:-main}" echo "Using loco-server ref: ${LOCO_REF}" echo "Using microinit ref: ${MICROINIT_REF}" diff --git a/Makefile b/Makefile index 57341e5..2a7d789 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ LOCAL_LOCO_BIN := ../bigfred/bin/loco-server-android-arm64 VALKEY_SO := $(NATIVE_PREBUILT)/libvalkey-server.so MICROINIT_SO := $(NATIVE_PREBUILT)/libmicroinit.so -BIGFRED_REF ?= main +BIGFRED_REF ?= master MICROINIT_REF ?= main VALKEY_REPO ?= dcc-bigfred/deps-android-valkey @@ -93,7 +93,7 @@ $(LOCO_SO): $(CI_SCRIPTS_DIR)/.ok GITHUB_REPO=dcc-bigfred/bigfred \ ARTIFACT_NAME=binaries \ FILES=loco-server-android-arm64:bin/libloco-server.so \ - "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(BIGFRED_REF)" "$$tmpdir/out.tar"; \ + "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(BIGFRED_REF)" "$$tmpdir/out.tar" && \ tar -xOf "$$tmpdir/out.tar" bin/libloco-server.so > "$@" @chmod 755 "$@" @echo "Wrote $@" @@ -113,7 +113,7 @@ $(MICROINIT_SO): $(CI_SCRIPTS_DIR)/.ok GITHUB_REPO=dcc-bigfred/microinit \ ARTIFACT_NAME=binaries-android-arm64 \ FILES=libmicroinit.so:bin/libmicroinit.so \ - "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(MICROINIT_REF)" "$$tmpdir/out.tar"; \ + "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(MICROINIT_REF)" "$$tmpdir/out.tar" && \ tar -xOf "$$tmpdir/out.tar" bin/libmicroinit.so > "$@" @chmod 755 "$@" @echo "Wrote $@" From 343b7fba90473060c9e5ba38ae6de788a5c8d445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= <372403+keskad@users.noreply.github.com> Date: Thu, 6 Aug 2026 16:07:12 +0200 Subject: [PATCH 3/5] ci: re-run native fetch against fixed .github@v2 tarball paths. Co-authored-by: Cursor From 086167b2cdf5dd2d6ec900cf4965e09ee69aca6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= <372403+keskad@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:52:15 +0200 Subject: [PATCH 4/5] Fetch native binaries via go run dcc-bigfred/common/cmd/fetch. Drop .ci-github clone/bootstrap; require Go on the host for tip/Release native fetches. Co-authored-by: Cursor --- Makefile | 50 ++++++++++++++++++-------------------------- app/build.gradle.kts | 39 +++++++++++++++++----------------- 2 files changed, 40 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index 2a7d789..6bddd86 100644 --- a/Makefile +++ b/Makefile @@ -8,10 +8,14 @@ # make loco-android — download libloco-server.so (GitHub tip/Release) # make valkey-android — download libvalkey-server.so from deps-android-valkey # make microinit-android — download libmicroinit.so (GitHub tip/Release) +# +# Native tip/Release fetch: go run github.com/dcc-bigfred/common/cmd/fetch@latest GRADLE ?= ./gradlew GRADLE_FLAGS ?= --quiet PYTHON ?= python3 +FETCH_PKG ?= github.com/dcc-bigfred/common/cmd/fetch@latest +export GOPROXY ?= direct APK_RELEASE := app/build/outputs/apk/release/app-release.apk APK_DEBUG := app/build/outputs/apk/debug/app-debug.apk @@ -30,13 +34,8 @@ BIGFRED_REF ?= master MICROINIT_REF ?= main VALKEY_REPO ?= dcc-bigfred/deps-android-valkey -CI_SCRIPTS_REPO ?= https://github.com/dcc-bigfred/.github.git -CI_SCRIPTS_REF ?= v2 -CI_SCRIPTS_DIR ?= .ci-github - .PHONY: help apk release test test-android debug clean import-models \ - loco-android valkey-android microinit-android native-prebuilt \ - ci-scripts ci-scripts-update + loco-android valkey-android microinit-android native-prebuilt help: @echo "Targets:" @@ -44,9 +43,9 @@ help: @echo " make loco-android Fetch $(LOCO_SO) (local bin or GitHub $(BIGFRED_REF))" @echo " make valkey-android Fetch $(VALKEY_SO) from $(VALKEY_REPO) latest release" @echo " make microinit-android Fetch $(MICROINIT_SO) from GitHub $(MICROINIT_REF)" - @echo " make ci-scripts Clone dcc-bigfred/.github @ $(CI_SCRIPTS_REF)" @echo "" @echo "Tip refs need GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN" + @echo "Binary fetch: go run $(FETCH_PKG)" import-models: $(PYTHON) "$(IMPORT_SCRIPT)" --out "$(IMPORT_OUT)" @@ -56,19 +55,6 @@ import-models: cp -a "$(IMPORT_OUT)/images" "$(ASSETS_MODELS)/images" @echo "Assets ready: $(ASSETS_MODELS)" -$(CI_SCRIPTS_DIR)/.ok: - @echo "Cloning $(CI_SCRIPTS_REPO) @ $(CI_SCRIPTS_REF) → $(CI_SCRIPTS_DIR)" - @rm -rf "$(CI_SCRIPTS_DIR)" - @git clone --depth 1 --branch "$(CI_SCRIPTS_REF)" "$(CI_SCRIPTS_REPO)" "$(CI_SCRIPTS_DIR)" \ - || { echo "error: failed to clone $(CI_SCRIPTS_REPO) @ $(CI_SCRIPTS_REF)"; exit 1; } - @touch "$@" - -ci-scripts: $(CI_SCRIPTS_DIR)/.ok - -ci-scripts-update: - rm -rf "$(CI_SCRIPTS_DIR)" - $(MAKE) "$(CI_SCRIPTS_DIR)/.ok" - ifdef FORCE .PHONY: force-clean-native force-clean-native: @@ -86,14 +72,16 @@ $(LOCO_SO): $(LOCAL_LOCO_BIN) @cp "$<" "$@" @echo "Using local $< → $@" else -$(LOCO_SO): $(CI_SCRIPTS_DIR)/.ok +$(LOCO_SO): + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } @mkdir -p "$(NATIVE_PREBUILT)" @tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ - GITHUB_REPO=dcc-bigfred/bigfred \ - ARTIFACT_NAME=binaries \ - FILES=loco-server-android-arm64:bin/libloco-server.so \ - "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(BIGFRED_REF)" "$$tmpdir/out.tar" && \ + go run "$(FETCH_PKG)" \ + --repo=dcc-bigfred/bigfred \ + --artifact=binaries \ + --files=loco-server-android-arm64:bin/libloco-server.so \ + "$(BIGFRED_REF)" "$$tmpdir/out.tar" && \ tar -xOf "$$tmpdir/out.tar" bin/libloco-server.so > "$@" @chmod 755 "$@" @echo "Wrote $@" @@ -106,14 +94,16 @@ $(VALKEY_SO): microinit-android: $(MICROINIT_SO) -$(MICROINIT_SO): $(CI_SCRIPTS_DIR)/.ok +$(MICROINIT_SO): + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } @mkdir -p "$(NATIVE_PREBUILT)" @tmpdir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmpdir"' EXIT; \ - GITHUB_REPO=dcc-bigfred/microinit \ - ARTIFACT_NAME=binaries-android-arm64 \ - FILES=libmicroinit.so:bin/libmicroinit.so \ - "$(CI_SCRIPTS_DIR)/scripts/fetch-github-binaries.sh" "$(MICROINIT_REF)" "$$tmpdir/out.tar" && \ + go run "$(FETCH_PKG)" \ + --repo=dcc-bigfred/microinit \ + --artifact=binaries-android-arm64 \ + --files=libmicroinit.so:bin/libmicroinit.so \ + "$(MICROINIT_REF)" "$$tmpdir/out.tar" && \ tar -xOf "$$tmpdir/out.tar" bin/libmicroinit.so > "$@" @chmod 755 "$@" @echo "Wrote $@" diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2f7e2b1..34d0d59 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -170,24 +170,12 @@ val fetchNativeBinaries by tasks.registering { return true } - fun runScript(script: File, vararg args: String) { - require(script.isFile) { "Missing script: ${script.absolutePath}" } - val pb = ProcessBuilder(listOf(script.absolutePath, *args)) - pb.directory(rootProject.projectDir) - pb.redirectErrorStream(true) - val env = pb.environment() - githubToken()?.let { env["GITHUB_TOKEN"] = it } - val proc = pb.start() - val output = proc.inputStream.bufferedReader().readText() - val code = proc.waitFor() - print(output) - if (code != 0) { - throw GradleException("Script ${script.name} failed (exit $code)") - } - } - - fun runMake(vararg targets: String) { - val pb = ProcessBuilder(listOf("make", *targets)) + fun runProcess( + command: List, + label: String, + extraEnv: Map = emptyMap(), + ) { + val pb = ProcessBuilder(command) pb.directory(rootProject.projectDir) pb.redirectErrorStream(true) val env = pb.environment() @@ -195,15 +183,28 @@ val fetchNativeBinaries by tasks.registering { env["GITHUB_TOKEN"] = it env["GH_TOKEN"] = it } + env.putAll(extraEnv) val proc = pb.start() val output = proc.inputStream.bufferedReader().readText() val code = proc.waitFor() print(output) if (code != 0) { - throw GradleException("make ${targets.joinToString(" ")} failed (exit $code)") + throw GradleException("$label failed (exit $code)") } } + fun runScript(script: File, vararg args: String) { + require(script.isFile) { "Missing script: ${script.absolutePath}" } + runProcess( + listOf(script.absolutePath, *args), + "Script ${script.name}", + ) + } + + fun runMake(vararg targets: String) { + runProcess(listOf("make", *targets), "make ${targets.joinToString(" ")}") + } + fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) { runScript(fetchScript.asFile, repo, assetName, dest.absolutePath) } From 11b79ddcc1410efed5fc50c3cb9f6bff3710e6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Damian=20K=C4=99ska?= <372403+keskad@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:09:58 +0200 Subject: [PATCH 5/5] Fetch Valkey via go run common/cmd/fetch. Drop the one-off release-asset shell script; Gradle stages through make. Co-authored-by: Cursor --- Makefile | 12 +++++- app/build.gradle.kts | 57 ++++++++++----------------- gradle.properties | 8 ++-- scripts/fetch-github-release-asset.sh | 57 --------------------------- 4 files changed, 35 insertions(+), 99 deletions(-) delete mode 100755 scripts/fetch-github-release-asset.sh diff --git a/Makefile b/Makefile index 6bddd86..54a5875 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,17 @@ endif valkey-android: $(VALKEY_SO) $(VALKEY_SO): - ./scripts/fetch-github-release-asset.sh "$(VALKEY_REPO)" libvalkey-server.so "$@" + @command -v go >/dev/null 2>&1 || { echo "error: go required for native fetch"; exit 1; } + @mkdir -p "$(NATIVE_PREBUILT)" + @tmpdir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmpdir"' EXIT; \ + go run "$(FETCH_PKG)" \ + --repo="$(VALKEY_REPO)" \ + --files=libvalkey-server.so:bin/libvalkey-server.so \ + latest-release "$$tmpdir/out.tar" && \ + tar -xOf "$$tmpdir/out.tar" bin/libvalkey-server.so > "$@" + @chmod 755 "$@" + @echo "Wrote $@" microinit-android: $(MICROINIT_SO) diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 34d0d59..15fb8c6 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -150,7 +150,6 @@ val fetchNativeBinaries by tasks.registering { val prebuilt = rootProject.layout.projectDirectory.dir("native-prebuilt/arm64-v8a") val localLoco = rootProject.layout.projectDirectory.dir("../bigfred/bin") .file("loco-server-android-arm64") - val fetchScript = rootProject.layout.projectDirectory.file("scripts/fetch-github-release-asset.sh") outputs.dir(jniOut) doLast { val outDir = jniOut.asFile @@ -163,13 +162,6 @@ val fetchNativeBinaries by tasks.registering { ?: System.getenv("GH_TOKEN") ?: System.getenv("GITHUB_TOKEN") - fun copyIfExists(src: File, destName: String): Boolean { - if (!src.isFile) return false - src.copyTo(File(outDir, destName), overwrite = true) - println("Staged $destName from ${src.absolutePath}") - return true - } - fun runProcess( command: List, label: String, @@ -193,41 +185,34 @@ val fetchNativeBinaries by tasks.registering { } } - fun runScript(script: File, vararg args: String) { - require(script.isFile) { "Missing script: ${script.absolutePath}" } - runProcess( - listOf(script.absolutePath, *args), - "Script ${script.name}", - ) - } - fun runMake(vararg targets: String) { runProcess(listOf("make", *targets), "make ${targets.joinToString(" ")}") } - fun fetchLatestReleaseAsset(repo: String, assetName: String, dest: File) { - runScript(fetchScript.asFile, repo, assetName, dest.absolutePath) + // Valkey: native-prebuilt → make valkey-android (GitHub latest-release via go fetch). + val valkeyDest = File(outDir, "libvalkey-server.so") + val valkeyCached = prebuilt.file("libvalkey-server.so").asFile + when { + valkeyCached.isFile -> { + valkeyCached.copyTo(valkeyDest, overwrite = true) + println("Staged libvalkey-server.so from ${valkeyCached.absolutePath}") + } + else -> { + runMake("valkey-android") + require(valkeyCached.isFile) { + "make valkey-android did not produce ${valkeyCached}" + } + valkeyCached.copyTo(valkeyDest, overwrite = true) + } } - - fun stageOrFetch(prebuiltName: String, repoProp: String, defaultRepo: String) { - val dest = File(outDir, prebuiltName) - if (copyIfExists(prebuilt.file(prebuiltName).asFile, prebuiltName)) return - val repo = (project.findProperty(repoProp) as String?) - ?.ifBlank { null } - ?: defaultRepo - fetchLatestReleaseAsset(repo, prebuiltName, dest) - val cache = prebuilt.file(prebuiltName).asFile - cache.parentFile.mkdirs() - dest.copyTo(cache, overwrite = true) + if (!valkeyDest.isFile || valkeyDest.length() < 1024) { + throw GradleException( + "libvalkey-server.so missing after fetch. " + + "Place it in native-prebuilt/arm64-v8a/ or set GITHUB_TOKEN and " + + "run 'make valkey-android'.", + ) } - // Valkey: prefer make-fetched native-prebuilt, else latest GitHub release. - stageOrFetch( - "libvalkey-server.so", - "depsAndroidValkeyRepo", - "dcc-bigfred/deps-android-valkey", - ) - // loco-server: local ../bigfred/bin → native-prebuilt → make loco-android (GitHub). val locoDest = File(outDir, "libloco-server.so") val locoCached = prebuilt.file("libloco-server.so").asFile diff --git a/gradle.properties b/gradle.properties index 4bf84eb..395f1fb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,9 +3,7 @@ android.useAndroidX=true kotlin.code.style=official android.nonTransitiveRClass=true -# Native loco-server / microinit for "BigFred on phone" (GitHub Actions tip or Releases). +# Native loco-server / microinit / valkey for "BigFred on phone" (GitHub tip or Releases). # Prefer local ../bigfred/bin/loco-server-android-arm64, then native-prebuilt/, -# then `make loco-android` / `make microinit-android` (see Makefile BIGFRED_REF / MICROINIT_REF). - -# Valkey Android prebuilt (latest GitHub release). -depsAndroidValkeyRepo=dcc-bigfred/deps-android-valkey +# then `make loco-android` / `make microinit-android` / `make valkey-android` +# (see Makefile BIGFRED_REF / MICROINIT_REF / VALKEY_REPO). diff --git a/scripts/fetch-github-release-asset.sh b/scripts/fetch-github-release-asset.sh deleted file mode 100755 index 4d817a2..0000000 --- a/scripts/fetch-github-release-asset.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -# Download a single asset from the latest GitHub release of a repo. -# Usage: ./scripts/fetch-github-release-asset.sh -# -# Auth (optional, needed for private repos): -# GITHUB_TOKEN / GH_TOKEN / BIGFRED_NATIVE_TOKEN -set -euo pipefail - -REPO="${1:?usage: $0 }" -ASSET="${2:?}" -OUT="${3:?}" - -mkdir -p "$(dirname "${OUT}")" - -TOKEN="${BIGFRED_NATIVE_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}" -AUTH=() -if [[ -n "${TOKEN}" ]]; then - AUTH=(-H "Authorization: Bearer ${TOKEN}") -fi - -API="https://api.github.com/repos/${REPO}/releases/latest" -echo "Resolving latest release of ${REPO}…" -json="$(curl -fsSL "${AUTH[@]}" -H "Accept: application/vnd.github+json" "${API}")" - -tag="$(printf '%s' "${json}" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("tag_name",""))')" -url="$(printf '%s' "${json}" | python3 -c ' -import json,sys -asset=sys.argv[1] -data=json.load(sys.stdin) -for a in data.get("assets") or []: - if a.get("name")==asset: - print(a.get("url") or "") - break -' "${ASSET}")" - -if [[ -z "${url}" ]]; then - echo "error: asset '${ASSET}' not found in latest release of ${REPO} (tag=${tag:-unknown})" >&2 - echo "Available assets:" >&2 - printf '%s' "${json}" | python3 -c 'import json,sys; [print(" -",a.get("name")) for a in (json.load(sys.stdin).get("assets") or [])]' >&2 - exit 1 -fi - -echo "Downloading ${ASSET} from ${REPO}@${tag}" -tmp="$(mktemp)" -cleanup() { rm -f "${tmp}"; } -trap cleanup EXIT - -# Asset API URL requires Accept: application/octet-stream -curl -fsSL "${AUTH[@]}" \ - -H "Accept: application/octet-stream" \ - -L \ - -o "${tmp}" \ - "${url}" - -cp -f "${tmp}" "${OUT}" -chmod 755 "${OUT}" -echo "Wrote ${OUT} ($(wc -c < "${OUT}") bytes) from ${REPO}@${tag}"