diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 14ef9c1..d2a76b8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -125,6 +125,18 @@ jobs: --prerelease="$prerelease" fi + # The installer of this release and its checksum, so an install can be + # pinned to a version and verified before it runs (issue #373). The + # asset clones this release; see scripts/build_installer_asset.py. + - name: Attach the installer and its checksum + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + tag='${{ steps.meta.outputs.tag }}' + python scripts/build_installer_asset.py "$tag" dist + gh release upload "$tag" dist/install.sh dist/SHA256SUMS --repo "$GITHUB_REPOSITORY" --clobber + - name: Summarise what was published run: | { diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f3ac6a..5468ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.34.64] - 2026-10-23 + +### Added +- **Each release carries its installer and a checksum.** The documented install was `bash <(curl …/main/install/install.sh)`: an unversioned URL with nothing to check the bytes against, so neither an operator nor a security policy could verify what was about to run as their user, and no one could say afterwards which program an Endpoint had run. The release workflow now attaches `install.sh` and `SHA256SUMS` to every release (built by `scripts/build_installer_asset.py`), and `install/README.md` documents downloading both for a version, `sha256sum -c SHA256SUMS`, then running it. The asset is `install/install.sh` of that tag with one line set, `INSTALLER_RELEASE`, so it installs its own release rather than the newest (`--ref` and `EP_REPO_REF` still choose another). The checksum of the installer that ran is recorded as `INSTALLER_SHA256` in `.env.install-state`, next to `INSTALLER_VERSION`. Not signed yet: a signature needs a key held by the organization, still to be decided. New tests in `tests/test_build_installer_asset.py` and `install/tests/test_default_release_ref.py`. Verified with a real install: the asset built for v0.34.63 passed `sha256sum -c`, cloned v0.34.63 although run from outside any checkout, and the Endpoint it installed answered healthy (#373). + ## [0.34.63] - 2026-10-23 ### Added diff --git a/api/config/swagger_settings.py b/api/config/swagger_settings.py index 5e6c02c..425df15 100644 --- a/api/config/swagger_settings.py +++ b/api/config/swagger_settings.py @@ -12,7 +12,7 @@ class Settings(BaseSettings): swagger_title: str = "API Documentation" swagger_description: str = "This is the API documentation." - swagger_version: str = "0.34.63" + swagger_version: str = "0.34.64" root_path: str = "" # API root path prefix (e.g., "/test" or "") is_public: bool = True metrics_endpoint: str = "https://federation.ndp.utah.edu/metrics/" diff --git a/docs/installing-with-the-script.md b/docs/installing-with-the-script.md index 043b12a..feaaaba 100644 --- a/docs/installing-with-the-script.md +++ b/docs/installing-with-the-script.md @@ -24,6 +24,10 @@ Endpoint up. Nothing is written or started until you confirm. copy the token from your user panel. The token is sent only to the Federation and is never stored. +To install a specific version and check the installer before it runs, use +the `install.sh` and `SHA256SUMS` attached to that release (see +[install/README.md](../install/README.md#verified-pinned-to-a-release)). + ## What the installer asks Run it and answer the prompts. Each has a short explanation on screen; the main diff --git a/install/README.md b/install/README.md index a629d5a..732f761 100644 --- a/install/README.md +++ b/install/README.md @@ -22,6 +22,25 @@ and `EP_REPO_URL` override the location and the repository. The version that ran is recorded as `INSTALLER_VERSION` in `.env.install-state` and in the [install log](#the-install-log). +### Verified, pinned to a release + +Each release (since 0.34.64) carries the installer and its checksum as assets: +`install.sh` and `SHA256SUMS`. Downloaded from a release, the installer installs +that release, and it can be checked before it runs: + +```bash +V=v0.34.64 # the release to install +curl -fsSLO "https://github.com/national-data-platform/ep-api/releases/download/$V/install.sh" +curl -fsSLO "https://github.com/national-data-platform/ep-api/releases/download/$V/SHA256SUMS" +sha256sum -c SHA256SUMS && bash install.sh [options] +``` + +The checksum protects against a corrupted or altered download, not against a +compromised GitHub account; the assets are not signed yet. The asset is +`install/install.sh` of that tag with one line set (`INSTALLER_RELEASE`, built +by `scripts/build_installer_asset.py`). The checksum of the installer that ran +is recorded as `INSTALLER_SHA256` in `.env.install-state`. + The installer needs `curl`, `python3`, `docker` and Docker Compose (`git` too when it has to clone). It renders `.env` from `example.env`, builds the image from the checkout and starts it with `docker compose up -d --build`, then waits diff --git a/install/install.sh b/install/install.sh index 0c850c4..23780a7 100755 --- a/install/install.sh +++ b/install/install.sh @@ -25,6 +25,12 @@ set -euo pipefail REPO_URL_DEFAULT="https://github.com/national-data-platform/ep-api.git" +# Empty here. In the install.sh attached to a release, which is checksummed +# in that release's SHA256SUMS, it names that release, so the downloaded +# installer installs its own version (scripts/build_installer_asset.py, +# issue #373). Keep it on a line of its own, exactly as it is. +INSTALLER_RELEASE="" + # -------------------------------------------------------------- # Bootstrap: work whether run from a checkout or piped from the web. # -------------------------------------------------------------- @@ -85,6 +91,9 @@ if [[ -z "$_here" || ! -f "$_here/example.env" ]]; then repo_ref="${_ref:-${EP_REPO_REF:-}}" if [[ -n "$repo_ref" ]]; then echo " Installer version: $repo_ref" + elif [[ -n "$INSTALLER_RELEASE" ]]; then + repo_ref="$INSTALLER_RELEASE" + echo " Installer version: $repo_ref (the release this installer came from)" else if ! _tags="$(git ls-remote --tags --refs "$repo_url" 'v*' 2>/dev/null)"; then echo "Could not list the releases of $repo_url. Check the network, or name a" >&2 @@ -2307,6 +2316,9 @@ fi # Which installer this is, for support and for a later status (issue #371). installer_version="$(git -C "$REPO_ROOT" describe --tags --always 2>/dev/null || echo unknown)" [[ "$dry_run" == "true" ]] || state_set INSTALLER_VERSION "$installer_version" +# And the checksum of the installer that ran, to compare with a release's +# SHA256SUMS (issue #373). +[[ "$dry_run" == "true" ]] || state_set INSTALLER_SHA256 "$(sha256sum "$REPO_ROOT/install/install.sh" | cut -d' ' -f1)" # The published port, which status and start report (issue #372). [[ "$dry_run" == "true" ]] || state_set EP_API_PORT "$ep_api_port" diff --git a/install/tests/test_default_release_ref.py b/install/tests/test_default_release_ref.py index 1c88c63..82533a8 100644 --- a/install/tests/test_default_release_ref.py +++ b/install/tests/test_default_release_ref.py @@ -179,6 +179,56 @@ def test_an_unreachable_repository_stops_with_a_message(tmp_path): assert "--ref" in result.stderr +def _asset(tmp_path, tag): + """The installer as a release publishes it (scripts/build_installer_asset.py).""" + import sys + + sys.path.insert(0, str(INSTALLER.parents[1])) + from scripts.build_installer_asset import build + + out = tmp_path / "release-asset" + build(tag, out, installer=INSTALLER) + return out / "install.sh" + + +def test_a_release_asset_installs_its_own_release(tmp_path, remote): + """Downloaded from v0.9.0, it installs v0.9.0 though v0.10.0 is newer.""" + asset = _asset(tmp_path, "v0.9.0") + env = { + **os.environ, + "EP_REPO_URL": str(remote), + "EP_INSTALL_DIR": str(tmp_path / "ndp-ep"), + } + env.pop("EP_REPO_REF", None) + + result = subprocess.run( + [BASH, str(asset)], capture_output=True, text=True, timeout=120, env=env + ) + + assert "RAN v0.9.0" in result.stdout, result.stdout + result.stderr + assert "the release this installer came from" in result.stdout + + +def test_ref_still_chooses_over_the_embedded_release(tmp_path, remote): + asset = _asset(tmp_path, "v0.9.0") + env = { + **os.environ, + "EP_REPO_URL": str(remote), + "EP_INSTALL_DIR": str(tmp_path / "ndp-ep"), + } + env.pop("EP_REPO_REF", None) + + result = subprocess.run( + [BASH, str(asset), "--ref", "main"], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + + assert "RAN main" in result.stdout + + def test_the_installed_version_is_recorded(tmp_path): """The install state says which installer ran (for support and status).""" root = tmp_path / "ep-api" @@ -214,3 +264,7 @@ def test_the_installed_version_is_recorded(tmp_path): assert result.returncode == 0, result.stdout + result.stderr state = (root / ".env.install-state").read_text(encoding="utf-8") assert "INSTALLER_VERSION=v1.2.3" in state.splitlines() + import hashlib + + digest = hashlib.sha256((root / "install" / "install.sh").read_bytes()).hexdigest() + assert f"INSTALLER_SHA256={digest}" in state.splitlines() diff --git a/scripts/build_installer_asset.py b/scripts/build_installer_asset.py new file mode 100644 index 0000000..e0de809 --- /dev/null +++ b/scripts/build_installer_asset.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Build the installer published with a release, and its checksum. + +The documented way to install was ``bash <(curl …/main/install/install.sh)``: +an unversioned URL, with nothing to verify the bytes against (issue #373). +Each release now carries ``install.sh`` and ``SHA256SUMS`` as assets, so an +operator can download the installer of one version, check it with +``sha256sum -c SHA256SUMS`` and run it. + +The asset is ``install/install.sh`` with one line changed: ``INSTALLER_RELEASE`` +names the release, so the asset clones that release rather than the newest one +(``--ref`` and ``EP_REPO_REF`` still choose another). Everything else is the +file as tagged. +""" + +import argparse +import hashlib +import re +import sys +from pathlib import Path + +DEFAULT_INSTALLER = Path("install/install.sh") +MARKER = 'INSTALLER_RELEASE=""' +_TAG_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.]+)?$") + + +def build(tag, out_dir, installer=DEFAULT_INSTALLER): + """Write ``install.sh`` and ``SHA256SUMS`` for ``tag`` into ``out_dir``. + + Parameters + ---------- + tag : str + The release tag, e.g. ``v0.34.64``. + out_dir : str or Path + Directory to write the two files into; created if missing. + installer : str or Path + The installer to publish. + + Returns + ------- + str + The SHA-256 of the published ``install.sh``. + + Raises + ------ + ValueError + When the tag is not a release tag, or the installer does not carry the + ``INSTALLER_RELEASE`` line exactly once. + """ + if not _TAG_RE.match(tag or ""): + raise ValueError(f"not a release tag: {tag!r}") + lines = Path(installer).read_text(encoding="utf-8").splitlines(keepends=True) + hits = [i for i, line in enumerate(lines) if line.rstrip("\n") == MARKER] + if len(hits) != 1: + raise ValueError( + f"{installer} must contain the line {MARKER} exactly once " + f"(found {len(hits)})" + ) + lines[hits[0]] = f'INSTALLER_RELEASE="{tag}"\n' + + out = Path(out_dir) + out.mkdir(parents=True, exist_ok=True) + asset = out / "install.sh" + asset.write_text("".join(lines), encoding="utf-8") + asset.chmod(0o755) + digest = hashlib.sha256(asset.read_bytes()).hexdigest() + (out / "SHA256SUMS").write_text(f"{digest} install.sh\n", encoding="utf-8") + return digest + + +def main(argv=None): + """Command-line entry point; returns the process exit code.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("tag", help="release tag, e.g. v0.34.64") + parser.add_argument("out_dir", help="where to write install.sh and SHA256SUMS") + parser.add_argument("--installer", default=str(DEFAULT_INSTALLER)) + args = parser.parse_args(argv) + try: + digest = build(args.tag, args.out_dir, installer=args.installer) + except ValueError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + print(f"{digest} install.sh ({args.tag})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_build_installer_asset.py b/tests/test_build_installer_asset.py new file mode 100644 index 0000000..511ea42 --- /dev/null +++ b/tests/test_build_installer_asset.py @@ -0,0 +1,80 @@ +"""Tests for the installer published with each release (issue #373).""" + +import hashlib +import re +from pathlib import Path + +import pytest + +from scripts.build_installer_asset import build, main + +REPO = Path(__file__).resolve().parents[1] +INSTALLER = REPO / "install" / "install.sh" +WORKFLOW = REPO / ".github" / "workflows" / "docker-publish.yml" + + +def test_the_asset_installs_its_own_release(tmp_path): + """Run from the release, it clones that release, not whatever is latest.""" + build("v1.2.3", tmp_path, installer=INSTALLER) + + asset = (tmp_path / "install.sh").read_text(encoding="utf-8") + assert 'INSTALLER_RELEASE="v1.2.3"' in asset + assert asset.count('INSTALLER_RELEASE="') == 1 + + +def test_the_rest_of_the_installer_is_unchanged(tmp_path): + build("v1.2.3", tmp_path, installer=INSTALLER) + + original = INSTALLER.read_text(encoding="utf-8").splitlines() + asset = (tmp_path / "install.sh").read_text(encoding="utf-8").splitlines() + different = [pair for pair in zip(original, asset) if pair[0] != pair[1]] + assert len(original) == len(asset) + assert different == [('INSTALLER_RELEASE=""', 'INSTALLER_RELEASE="v1.2.3"')] + + +def test_sha256sums_matches_the_asset(tmp_path): + build("v1.2.3", tmp_path, installer=INSTALLER) + + digest = hashlib.sha256((tmp_path / "install.sh").read_bytes()).hexdigest() + assert (tmp_path / "SHA256SUMS").read_text() == f"{digest} install.sh\n" + + +def test_the_asset_is_executable(tmp_path): + build("v1.2.3", tmp_path, installer=INSTALLER) + + assert (tmp_path / "install.sh").stat().st_mode & 0o111 + + +@pytest.mark.parametrize("tag", ["1.2.3", "v1.2", "main", "v1.2.3; rm -rf /", ""]) +def test_only_a_release_tag_is_embedded(tmp_path, tag): + with pytest.raises(ValueError): + build(tag, tmp_path, installer=INSTALLER) + + +def test_a_prerelease_tag_is_accepted(tmp_path): + build("v1.3.0-rc1", tmp_path, installer=INSTALLER) + + assert 'INSTALLER_RELEASE="v1.3.0-rc1"' in (tmp_path / "install.sh").read_text() + + +def test_an_installer_without_the_marker_is_refused(tmp_path): + """A rename must fail the release, not ship an asset that tracks latest.""" + fake = tmp_path / "install.sh.src" + fake.write_text("#!/bin/bash\necho hi\n", encoding="utf-8") + + with pytest.raises(ValueError): + build("v1.2.3", tmp_path / "out", installer=fake) + + +def test_the_command_line(tmp_path, capsys): + assert main(["v1.2.3", str(tmp_path), "--installer", str(INSTALLER)]) == 0 + assert (tmp_path / "SHA256SUMS").exists() + + +def test_the_release_workflow_publishes_them(): + text = WORKFLOW.read_text(encoding="utf-8") + + assert "scripts/build_installer_asset.py" in text + upload = re.search(r"gh release upload .*--clobber", text) + assert upload, "the assets are never attached to the release" + assert text.index("gh release create") < text.index("gh release upload")