Installer: keep an install log of every run (#374)

.gitignore

@@ -12,3 +12,4 @@ setup_info.txt
 tests_backup/
 ui/node_modules/
 ui/build/
+/install.log

CHANGELOG.md

@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.34.59] - 2026-10-23
+
+### Added
+- **Every installer run leaves a log.** The installer only wrote to the terminal, so a run that failed unattended, or ten minutes in, left nothing on the machine to say what it was asked to do or where it stopped. Each run is now appended to `install.log` in the checkout (`~/ndp-ep` or `~/ndp-ep-<instance>` from the web), mode 600, starting with a header (time, `git describe` of the installer, arguments with `--ckan-password`, `--ckan-api-key`, `--s3-access-key` and `--s3-secret-key` masked, user and host, Federation URL) and followed by everything the run printed, without colour codes. Re-runs append. A failure ends with `install log: <path>`. `--log-file <path>` writes it elsewhere and `--log-file /dev/null` turns it off; a log that cannot be written is reported and the run carries on. What the terminal shows, on stdout and stderr, is unchanged. New tests in `install/tests/test_install_log.py`. Verified with a real install of an Endpoint on this machine (Docker build included): the log held the header, the build output and the health check, without colour codes, and a failing run against a port already in use ended with the log's path on screen and in the file (#374).
+
 ## [0.34.58] - 2026-10-20
 
 ### Fixed

api/config/swagger_settings.py

@@ -12,7 +12,7 @@ class Settings(BaseSettings):
 
     swagger_title: str = "API Documentation"
     swagger_description: str = "This is the API documentation."
-    swagger_version: str = "0.34.58"
+    swagger_version: str = "0.34.59"
     root_path: str = ""  # API root path prefix (e.g., "/test" or "")
     is_public: bool = True
     metrics_endpoint: str = "https://federation.ndp.utah.edu/metrics/"

docs/installing-with-the-script.md

@@ -164,6 +164,9 @@ A URL you enter without a scheme gets `https://` added automatically.
   platform — or with your username and password.
 - As the Endpoint's administrator you'll see the management areas (datasets,
   services, organizations, access requests).
+- Each run of the installer is recorded in `install.log` next to the
+  Endpoint's files (`~/ndp-ep` by default). If an installation fails, that is
+  the file to send with a support request; the failure message gives its path.
 
 ## Running it again / other options
 

install/README.md

@@ -231,6 +231,7 @@ re-run to upgrade it: it only joins with `--netbird`. Affinities stays opt-in
 | `--affinities` | turn the Affinities integration on from the Federation configuration |
 | `--no-start` | write everything, bring nothing up; prints the command to start it |
 | `--yes` | no prompts at all (see [Without prompts](#without-prompts)), and overwrite an existing `.env` without asking |
+| `--log-file <path>` | append the run to `<path>` instead of `install.log` in the checkout; `/dev/null` keeps no log (see [The install log](#the-install-log)) |
 | `-h`, `--help` | usage |
 
 The underscore spellings are the ones the platform's create-endpoint page
@@ -310,6 +311,18 @@ and prove nothing. Checking the key needs a username, so with `--ckan-url`
 pass `--ckan-sysadmin <name>` to enable it; without it the installer says the
 key went unverified rather than implying it passed.
 
+## The install log
+
+Every run is appended to `install.log` in the checkout (`~/ndp-ep` or
+`~/ndp-ep-<instance>` when installed from the web), readable by its owner only.
+Each run starts with a header — time, installer version (`git describe`),
+arguments, user and host, Federation — followed by everything the run printed,
+without colour codes. The values of `--ckan-password`, `--ckan-api-key`,
+`--s3-access-key` and `--s3-secret-key` are masked in the header; prompts that
+hide what is typed keep hiding it. Re-runs append, so a failed attempt and the
+retry are both there, and a failure ends with the path of the log. A log that
+cannot be written is reported and the install carries on.
+
 ## How it works
 
 **It never edits files in this repository.** The published port is set with

install/install.sh

@@ -149,14 +149,22 @@ assume_yes="false"
 # The Federation has no field for a remote-execution URL, so when registration
 # captures one it is applied straight to the .env after the config is fetched.
 reg_rexec_url=""
+# Where this run is recorded (issue #374). Empty means install.log next to the
+# checkout; /dev/null turns the record off. log_active is set once it is open.
+log_file=""
+log_active=""
 
 RED=$'\033[0;31m'; GREEN=$'\033[0;32m'; YELLOW=$'\033[1;33m'
 BLUE=$'\033[0;34m'; NC=$'\033[0m'
 
 info()    { echo "${BLUE}[info]${NC} $*"; }
 ok()      { echo "${GREEN}[ ok ]${NC} $*"; }
 warn()    { echo "${YELLOW}[warn]${NC} $*" >&2; }
-fail()    { echo "${RED}[fail]${NC} $*" >&2; exit 1; }
+fail()    {
+  echo "${RED}[fail]${NC} $*" >&2
+  [[ -n "$log_active" ]] && echo "       install log: $log_active" >&2
+  exit 1
+}
 step()    { echo; echo "${GREEN}==>${NC} $*"; }
 
 banner() {
@@ -268,6 +276,8 @@ have:
   --dry-run               Render .env and run the checks, start nothing
   --no-start              Write everything, do not bring the stack up
   --yes                   Do not prompt before overwriting an existing .env
+  --log-file <path>       Append this run to <path>. Default: install.log in
+                          the checkout. /dev/null keeps no log.
   -h, --help              This message
 
 The configuration id comes from registering the Endpoint with the Federation.
@@ -276,6 +286,9 @@ provider) are applied on top of the documented defaults in example.env.
 USAGE
 }
 
+# As given, for the install log; the secrets among them are masked there.
+cli_args=("$@")
+
 while [[ $# -gt 0 ]]; do
   case "$1" in
     # The underscore spellings are what the platform's "create endpoint"
@@ -318,6 +331,7 @@ while [[ $# -gt 0 ]]; do
     --dry-run)         dry_run="true"; shift ;;
     --no-start)        start="false"; shift ;;
     --yes)             assume_yes="true"; shift ;;
+    --log-file)        log_file="${2:-}"; shift 2 ;;
     -h|--help)         usage; exit 0 ;;
     *)                 fail "Unknown option: $1 (try --help)" ;;
   esac
@@ -333,6 +347,63 @@ esac
 [[ "$backend" == "none" || "$backend" == "mongodb" || "$backend" == "ckan" ]] \
   || fail "--backend must be none, mongodb or ckan (got: $backend)"
 
+# --------------------------------------------------------------
+# The install log (issue #374)
+# --------------------------------------------------------------
+# Everything the run prints is also appended to a file, so a run that fails
+# unattended, or ten minutes in, leaves a record of what it was asked to do and
+# where it stopped. Appended, not truncated: a failed attempt and the retry
+# that followed are both kept. The terminal sees exactly what it did before.
+finish_log() { :; }
+[[ -n "$log_file" ]] || log_file="$REPO_ROOT/install.log"
+if [[ "$log_file" != "/dev/null" ]]; then
+  if ( umask 077; : >> "$log_file" ) 2>/dev/null; then
+    chmod 600 "$log_file" 2>/dev/null || true
+    shown_args=()
+    for ((_i = 0; _i < ${#cli_args[@]}; _i++)); do
+      shown_args+=("$(printf '%q' "${cli_args[$_i]}")")
+      case "${cli_args[$_i]}" in
+        --ckan-password|--ckan-api-key|--s3-access-key|--s3-secret-key)
+          if ((_i + 1 < ${#cli_args[@]})); then
+            shown_args+=("***"); _i=$((_i + 1))
+          fi ;;
+      esac
+    done
+    {
+      echo
+      echo "=== $(date -u '+%Y-%m-%dT%H:%M:%SZ') install.sh run"
+      echo "    version: $(git -C "$REPO_ROOT" describe --tags --always 2>/dev/null || echo unknown)"
+      echo "    args: ${shown_args[*]}"
+      echo "    user: $(id -un 2>/dev/null || echo unknown)@$(hostname 2>/dev/null || echo unknown)"
+      echo "    federation: $federation_url"
+    } >> "$log_file"
+    exec 5>&1 6>&2
+    exec > >(tee -a "$log_file")
+    log_out_pid=$!
+    exec 2> >(tee -a "$log_file" >&6)
+    log_err_pid=$!
+    log_active="$log_file"
+    # On the way out: hand the terminal back, let both copies finish writing,
+    # then strip the colour codes, which only make sense on a terminal.
+    finish_log() {
+      [[ -n "$log_active" ]] || return 0
+      exec 1>&5 2>&6
+      # Bounded: a daemon started during the run could still hold the pipe,
+      # and the record is not worth hanging the exit for.
+      local _w
+      for _w in $(seq 1 50); do
+        kill -0 "$log_out_pid" 2>/dev/null || kill -0 "$log_err_pid" 2>/dev/null || break
+        sleep 0.1
+      done
+      sed -i 's/\x1b\[[0-9;]*m//g' "$log_active" 2>/dev/null || true
+      log_active=""
+    }
+    trap finish_log EXIT
+  else
+    warn "Cannot write the install log at $log_file; carrying on without it."
+  fi
+fi
+
 banner
 
 # Which Federation this run will talk to, before anything happens. The script
@@ -554,7 +625,7 @@ with_scheme() {
 # Collect the values that will override the documented defaults.
 # --------------------------------------------------------------
 overrides_file="$(mktemp)"
-trap 'rm -f "$overrides_file" "${config_file:-}"' EXIT
+trap 'rm -f "$overrides_file" "${config_file:-}"; finish_log' EXIT
 echo '{}' > "$overrides_file"
 
 put() {

install/tests/test_install_log.py

@@ -0,0 +1,170 @@
+"""
+Every installer run leaves a log behind (issue #374).
+
+Before, everything went to the terminal only: a run that failed ten minutes in,
+unattended, left nothing on the machine to say where it stopped. Now the whole
+run is appended to ``install.log`` next to the checkout (or to ``--log-file``),
+with a header saying which installer ran, with which arguments and against
+which Federation, and a failure names the log.
+
+Each run is stopped at the prerequisite check by a ``docker`` that refuses to
+answer, which is enough to see the header, the output and the failure land in
+the file.
+"""
+
+import os
+import re
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO = Path(__file__).resolve().parents[2]
+BASH = shutil.which("bash")
+
+pytestmark = pytest.mark.skipif(BASH is None or os.name == "nt", reason="needs bash")
+
+ANSI = re.compile(r"\x1b\[[0-9;]*m")
+
+
+@pytest.fixture
+def checkout(tmp_path):
+    """A checkout of the installer plus a docker that never answers."""
+    root = tmp_path / "ep-api"
+    shutil.copytree(REPO / "install", root / "install")
+    shutil.copy(REPO / "example.env", root / "example.env")
+    shutil.copy(REPO / "docker-compose.yml", root / "docker-compose.yml")
+    stubs = tmp_path / "bin"
+    stubs.mkdir()
+    (stubs / "docker").write_text("#!/bin/sh\nexit 1\n", encoding="utf-8")
+    (stubs / "docker").chmod(0o755)
+    return {"root": root, "stubs": stubs, "tmp": tmp_path}
+
+
+def _run(checkout, *args):
+    return subprocess.run(
+        [BASH, str(checkout["root"] / "install" / "install.sh"), *args, "--dry-run"],
+        capture_output=True,
+        text=True,
+        timeout=120,
+        env={
+            **os.environ,
+            "PATH": f"{checkout['stubs']}{os.pathsep}{os.environ['PATH']}",
+        },
+        stdin=subprocess.DEVNULL,
+    )
+
+
+def test_a_run_is_logged_next_to_the_checkout(checkout):
+    result = _run(checkout, "--config-id", "FAKE", "--env", "test")
+
+    log = checkout["root"] / "install.log"
+    assert result.returncode != 0
+    assert log.exists(), result.stdout + result.stderr
+    text = log.read_text(encoding="utf-8")
+    assert "Federation: https://federation.ndp.utah.edu/test" in text
+
+
+def test_the_header_says_what_ran(checkout):
+    _run(checkout, "--config-id", "FAKE", "--env", "test")
+
+    text = (checkout["root"] / "install.log").read_text(encoding="utf-8")
+    assert "install.sh run" in text
+    assert "args: --config-id FAKE --env test --dry-run" in text
+    assert "federation: https://federation.ndp.utah.edu/test" in text
+    assert "version:" in text
+    assert "user:" in text
+
+
+def test_the_failure_lands_in_the_log_and_names_it(checkout):
+    result = _run(checkout, "--config-id", "FAKE")
+
+    log = checkout["root"] / "install.log"
+    assert f"install log: {log}" in result.stderr
+    text = log.read_text(encoding="utf-8")
+    assert "[fail]" in text
+
+
+def test_the_log_has_no_colour_codes(checkout):
+    _run(checkout, "--config-id", "FAKE")
+
+    text = (checkout["root"] / "install.log").read_text(encoding="utf-8")
+    assert not ANSI.search(text)
+
+
+def test_a_rerun_appends_rather_than_truncates(checkout):
+    _run(checkout, "--config-id", "FIRST")
+    _run(checkout, "--config-id", "SECOND")
+
+    text = (checkout["root"] / "install.log").read_text(encoding="utf-8")
+    assert "--config-id FIRST" in text
+    assert "--config-id SECOND" in text
+    assert text.count("install.sh run") == 2
+
+
+def test_secrets_given_as_arguments_are_not_logged(checkout):
+    _run(
+        checkout,
+        "--backend",
+        "ckan",
+        "--ckan-password",
+        "pw-Zq81",
+        "--ckan-api-key",
+        "ak-Zq82",
+        "--s3-endpoint",
+        "s3.example.invalid",
+        "--s3-access-key",
+        "ac-Zq83",
+        "--s3-secret-key",
+        "sk-Zq84",
+    )
+
+    text = (checkout["root"] / "install.log").read_text(encoding="utf-8")
+    for secret in ("pw-Zq81", "ak-Zq82", "ac-Zq83", "sk-Zq84"):
+        assert secret not in text
+    assert "--ckan-password ***" in text
+
+
+def test_the_log_is_readable_by_its_owner_only(checkout):
+    _run(checkout, "--config-id", "FAKE")
+
+    mode = (checkout["root"] / "install.log").stat().st_mode & 0o777
+    assert mode == 0o600
+
+
+def test_log_file_chooses_where_it_goes(checkout):
+    chosen = checkout["tmp"] / "elsewhere" / "run.log"
+    chosen.parent.mkdir()
+
+    result = _run(checkout, "--config-id", "FAKE", "--log-file", str(chosen))
+
+    assert chosen.exists()
+    assert "Federation:" in chosen.read_text(encoding="utf-8")
+    assert not (checkout["root"] / "install.log").exists()
+    assert f"install log: {chosen}" in result.stderr
+
+
+def test_dev_null_turns_the_log_off(checkout):
+    result = _run(checkout, "--config-id", "FAKE", "--log-file", "/dev/null")
+
+    assert not (checkout["root"] / "install.log").exists()
+    assert "install log:" not in result.stderr
+    assert "Federation:" in result.stdout
+
+
+def test_terminal_output_is_unchanged(checkout):
+    """The log is a copy: what the operator sees still arrives, on both streams."""
+    result = _run(checkout, "--config-id", "FAKE", "--env", "test")
+
+    assert "Federation: https://federation.ndp.utah.edu/test" in result.stdout
+    assert "[fail]" in result.stderr
+
+
+def test_an_unwritable_log_does_not_stop_the_install(checkout):
+    blocked = checkout["tmp"] / "no-such-dir" / "run.log"
+
+    result = _run(checkout, "--config-id", "FAKE", "--log-file", str(blocked))
+
+    assert "Cannot write the install log" in result.stderr
+    assert "Federation:" in result.stdout