diff --git a/CHANGELOG.md b/CHANGELOG.md index 837ebc9..4301452 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.34.60] - 2026-10-23 + +### Added +- **Building and starting has a time limit.** The installer ran `docker compose up -d --build`, for the Endpoint and for the CKAN it sets up, with no limit, so a pull stalled on a registry, a Docker Hub rate limit or a proxy kept the run waiting for ever after every earlier step had reported success, and an unattended install just looked busy. Both now run under `--compose-timeout` (minutes, or seconds with an `s` suffix; 30 minutes by default, `0` for no limit), print a line every minute while they work, and when the limit runs out the run stops saying what to check (reaching the registry with `docker pull`, the rate limit, a proxy) and that running it again carries on. A compose that fails outright is reported as such, with its exit code, instead of ending the run without a message. Without `timeout` (coreutils) there is no limit and the installer says so. New tests in `install/tests/test_compose_timeout.py`. Verified with a real install on this machine: with `--compose-timeout 2s` the image build was stopped after 3 seconds with that message, and running the installer again without the flag finished the install, healthy (#376). + ## [0.34.59] - 2026-10-23 ### Added diff --git a/api/config/swagger_settings.py b/api/config/swagger_settings.py index a3a2499..eafb232 100644 --- a/api/config/swagger_settings.py +++ b/api/config/swagger_settings.py @@ -12,7 +12,7 @@ class Settings(BaseSettings): swagger_title: str = "API Documentation" swagger_description: str = "This is the API documentation." - swagger_version: str = "0.34.59" + swagger_version: str = "0.34.60" root_path: str = "" # API root path prefix (e.g., "/test" or "") is_public: bool = True metrics_endpoint: str = "https://federation.ndp.utah.edu/metrics/" diff --git a/install/README.md b/install/README.md index 115c741..06ff5fe 100644 --- a/install/README.md +++ b/install/README.md @@ -232,6 +232,7 @@ re-run to upgrade it: it only joins with `--netbird`. Affinities stays opt-in | `--no-start` | write everything, bring nothing up; prints the command to start it | | `--yes` | no prompts at all (see [Without prompts](#without-prompts)), and overwrite an existing `.env` without asking | | `--log-file ` | append the run to `` instead of `install.log` in the checkout; `/dev/null` keeps no log (see [The install log](#the-install-log)) | +| `--compose-timeout ` | give up when building and starting a stack takes more than `` minutes (`s` for seconds); default `30`, `0` for no limit | | `-h`, `--help` | usage | The underscore spellings are the ones the platform's create-endpoint page @@ -323,6 +324,16 @@ hide what is typed keep hiding it. Re-runs append, so a failed attempt and the retry are both there, and a failure ends with the path of the log. A log that cannot be written is reported and the install carries on. +## Time limit on building and starting + +`docker compose up -d --build`, for the Endpoint and for the CKAN the installer +sets up, runs under a time limit: 30 minutes unless `--compose-timeout` says +otherwise. While it runs, a line every minute says it is still working, so a +slow first build is not mistaken for a hang. A pull stalled on a registry, a +Docker Hub rate limit or a proxy then stops the run with what to check instead +of waiting for ever; running the installer again carries on from there. Without +the `timeout` command (coreutils) there is no limit, and the installer says so. + ## How it works **It never edits files in this repository.** The published port is set with diff --git a/install/install.sh b/install/install.sh index 3388dcd..3a2b67f 100755 --- a/install/install.sh +++ b/install/install.sh @@ -153,6 +153,9 @@ reg_rexec_url="" # checkout; /dev/null turns the record off. log_active is set once it is open. log_file="" log_active="" +# How long one "docker compose up --build" may take before the run gives up +# (issue #376): minutes, or seconds with an "s" suffix; 0 means no limit. +compose_timeout="30" RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[1;33m' BLUE=$'\033[0;34m'; NC=$'\033[0m' @@ -278,6 +281,9 @@ have: --yes Do not prompt before overwriting an existing .env --log-file Append this run to . Default: install.log in the checkout. /dev/null keeps no log. + --compose-timeout Give up when building and starting a stack takes + longer than minutes (or s seconds). Default: + 30. 0 means no limit. -h, --help This message The configuration id comes from registering the Endpoint with the Federation. @@ -332,6 +338,7 @@ while [[ $# -gt 0 ]]; do --no-start) start="false"; shift ;; --yes) assume_yes="true"; shift ;; --log-file) log_file="${2:-}"; shift 2 ;; + --compose-timeout) compose_timeout="${2-}"; shift 2 ;; -h|--help) usage; exit 0 ;; *) fail "Unknown option: $1 (try --help)" ;; esac @@ -347,6 +354,9 @@ esac [[ "$backend" == "none" || "$backend" == "mongodb" || "$backend" == "ckan" ]] \ || fail "--backend must be none, mongodb or ckan (got: $backend)" +[[ "$compose_timeout" =~ ^[0-9]+s?$ ]] \ + || fail "--compose-timeout must be a number of minutes, or of seconds ending in s (got: '$compose_timeout')" + # -------------------------------------------------------------- # The install log (issue #374) # -------------------------------------------------------------- @@ -430,6 +440,66 @@ docker info >/dev/null 2>&1 || fail "Cannot talk to the Docker daemon. Is it run [[ -f "$REPO_ROOT/example.env" ]] || fail "example.env not found at $REPO_ROOT — run this from a checkout of the repository." ok "curl, python3, docker and compose are available" +# -------------------------------------------------------------- +# A time limit on building and starting (issue #376) +# -------------------------------------------------------------- +# "docker compose up --build" pulls and builds, and a pull stalled on a +# registry, a rate limit or a proxy never returns: the run used to wait for +# ever, with every earlier step reported as done. It now gives up after +# --compose-timeout and says what to check; a heartbeat every minute tells a +# slow first build from a hang. +if [[ "$compose_timeout" == "0" || "$compose_timeout" == "0s" ]]; then + compose_limit="" + info "Docker Compose time limit: none" +elif ! command -v timeout >/dev/null 2>&1; then + compose_limit="" + warn "'timeout' is not installed, so building and starting have no time limit." +elif [[ "$compose_timeout" == *s ]]; then + compose_limit="$compose_timeout" + info "Docker Compose time limit: ${compose_timeout}" +else + compose_limit="$((compose_timeout * 60))s" + info "Docker Compose time limit: ${compose_timeout} min" +fi + +# compose_up WHAT DIR COMMAND... -- runs COMMAND in DIR within the limit. +# Returns its status; 124 means the limit ran out. +compose_up() { + local what="$1" dir="$2"; shift 2 + local started=$SECONDS rc=0 beat + # The sleep is waited on rather than run in the foreground so that stopping + # the heartbeat stops it too: left behind, it would hold the terminal (and + # the install log's pipe) open for up to a minute after the run ended. + ( + nap="" + trap 'kill "$nap" 2>/dev/null; exit 0' TERM + while :; do + sleep 60 & nap=$! + wait "$nap" + info "Still building and starting $what ($(( (SECONDS - started) / 60 )) min so far)" + done + ) & + beat=$! + if [[ -n "$compose_limit" ]]; then + (cd "$dir" && timeout -k 30 "$compose_limit" "$@") || rc=$? + else + (cd "$dir" && "$@") || rc=$? + fi + kill "$beat" 2>/dev/null || true + wait "$beat" 2>/dev/null || true + return "$rc" +} + +compose_stalled() { + local what="$1" dir="$2" + fail "Building and starting $what did not finish within ${compose_limit}. + A pull or build is probably stalled. Check that this machine reaches the + registry (try: docker pull python:3.13-slim), Docker Hub's rate limit + (docker login helps), and any proxy; then run the installer again, which + picks up where it stopped. A slow machine can allow more time with + --compose-timeout . To look: cd $dir && ${COMPOSE[*]} logs" +} + # -------------------------------------------------------------- # Asking # -------------------------------------------------------------- @@ -1697,8 +1767,10 @@ else ok "Ports $ckan_ssl_port, $ckan_http_port and $ckan_app_port are free" info "Building and starting CKAN — this takes several minutes the first time" - (cd "$ckan_dir" && "${COMPOSE[@]}" up -d --build) \ - || fail "CKAN failed to start. Inspect it with: cd $ckan_dir && ${COMPOSE[*]} logs" + rc=0 + compose_up CKAN "$ckan_dir" "${COMPOSE[@]}" up -d --build || rc=$? + [[ $rc -eq 124 ]] && compose_stalled CKAN "$ckan_dir" + [[ $rc -eq 0 ]] || fail "CKAN failed to start. Inspect it with: cd $ckan_dir && ${COMPOSE[*]} logs" step "Waiting for CKAN to become ready" ckan_cid="" @@ -1982,7 +2054,11 @@ cd "$REPO_ROOT" # making the mismatch look like a configuration problem. Unchanged sources # come from the layer cache, so this costs little after the first run. info "Building the image from this checkout; the first run takes a few minutes." -EP_API_PORT="$ep_api_port" "${COMPOSE[@]}" "${profile_args[@]}" up -d --build +rc=0 +compose_up "the Endpoint" "$REPO_ROOT" \ + env EP_API_PORT="$ep_api_port" "${COMPOSE[@]}" "${profile_args[@]}" up -d --build || rc=$? +[[ $rc -eq 124 ]] && compose_stalled "the Endpoint" "$REPO_ROOT" +[[ $rc -eq 0 ]] || fail "Docker Compose could not start the Endpoint (exit $rc). Inspect it with: cd $REPO_ROOT && ${COMPOSE[*]} logs" # -------------------------------------------------------------- step "Verifying the Endpoint answers" diff --git a/install/tests/test_compose_timeout.py b/install/tests/test_compose_timeout.py new file mode 100644 index 0000000..4dbbdab --- /dev/null +++ b/install/tests/test_compose_timeout.py @@ -0,0 +1,144 @@ +""" +A stalled ``docker compose up`` no longer hangs the installer (issue #376). + +The start step and the CKAN build ran ``docker compose up -d --build`` with no +limit, so a pull stuck on a registry, a rate limit or a proxy kept the run +waiting for ever after every earlier step had reported success. Both now run +under ``--compose-timeout`` (30 minutes unless given), and running out of time +stops the run with what to check. + +``docker`` is a stub whose ``compose up`` never returns (or fails at once), so +nothing is pulled or built. +""" + +import os +import shutil +import socket +import subprocess +import time +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +BASH = shutil.which("bash") + +pytestmark = pytest.mark.skipif( + BASH is None or os.name == "nt" or shutil.which("timeout") is None, + reason="needs bash and coreutils timeout", +) + + +def _free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _checkout(tmp_path, up_behaviour): + root = tmp_path / "ep-api" + shutil.copytree(REPO / "install", root / "install") + shutil.copy(REPO / "example.env", root / "example.env") + shutil.copy(REPO / "docker-compose.yml", root / "docker-compose.yml") + stubs = tmp_path / "bin" + stubs.mkdir() + (stubs / "docker").write_text( + "#!/bin/sh\n" + f'echo "docker $*" >> "{tmp_path}/docker.log"\n' + 'case " $* " in\n' + f" *' up '*) {up_behaviour} ;;\n" + "esac\n" + "exit 0\n", + encoding="utf-8", + ) + (stubs / "docker").chmod(0o755) + return root, stubs + + +def _install(root, stubs, *args): + started = time.monotonic() + result = subprocess.run( + [ + BASH, + str(root / "install" / "install.sh"), + "--backend", + "none", + "--no-netbird", + "--ep-api-port", + str(_free_port()), + "--yes", + *args, + ], + capture_output=True, + text=True, + timeout=120, + env={**os.environ, "PATH": f"{stubs}{os.pathsep}{os.environ['PATH']}"}, + stdin=subprocess.DEVNULL, + ) + return result, time.monotonic() - started + + +def test_a_stalled_compose_up_stops_the_run(tmp_path): + root, stubs = _checkout(tmp_path, "exec sleep 300") + + result, took = _install(root, stubs, "--compose-timeout", "3s") + + assert result.returncode != 0 + assert took < 60 + assert "did not finish within 3s" in result.stderr + + +def test_the_timeout_message_says_what_to_check(tmp_path): + root, stubs = _checkout(tmp_path, "exec sleep 300") + + result, _ = _install(root, stubs, "--compose-timeout", "3s") + + assert "docker pull" in result.stderr + assert "rate limit" in result.stderr + assert "--compose-timeout" in result.stderr + + +def test_a_compose_that_fails_is_not_reported_as_a_timeout(tmp_path): + root, stubs = _checkout(tmp_path, "exit 3") + + result, _ = _install(root, stubs, "--compose-timeout", "30s") + + assert result.returncode != 0 + assert "did not finish" not in result.stderr + assert "could not start the Endpoint" in result.stderr + + +@pytest.mark.parametrize("value", ["abc", "-5", "10x", ""]) +def test_a_malformed_timeout_is_refused(tmp_path, value): + root, stubs = _checkout(tmp_path, "exit 0") + + result, _ = _install(root, stubs, "--compose-timeout", value) + + assert result.returncode != 0 + assert "--compose-timeout" in result.stderr + + +def test_minutes_are_the_unit_without_a_suffix(tmp_path): + root, stubs = _checkout(tmp_path, "exit 3") + + result, _ = _install(root, stubs, "--compose-timeout", "45") + + assert "--compose-timeout must be" not in result.stderr + assert "Docker Compose time limit: 45 min" in result.stdout + + +def test_thirty_minutes_unless_given(tmp_path): + root, stubs = _checkout(tmp_path, "exit 3") + + result, _ = _install(root, stubs) + + assert "Docker Compose time limit: 30 min" in result.stdout + + +def test_zero_means_no_limit(tmp_path): + root, stubs = _checkout(tmp_path, "exit 3") + + result, _ = _install(root, stubs, "--compose-timeout", "0") + + assert "Docker Compose time limit: none" in result.stdout + assert "could not start the Endpoint" in result.stderr