diff --git a/AGENTS.md b/AGENTS.md index a0e4eb0e7..bc77854c5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,7 @@ recent host Ruby (4.x), so the site is built in a `ruby:3.3` container and the - `builder/apply-patches.sh` strips the `patches/` prefix and applies each patch into the matching upstream source directory. - `files/` is the rootfs overlay for the final image. `files/etc/uci-defaults` holds first-boot defaults. - Runtime web stack: **nginx** serves `ns-ui` from `/www-ns` and proxies `/api/` → **ns-api-server** on `127.0.0.1:8090`; `ns-api-server` handles auth/JWT and forwards calls to ubus/rpcd handlers. -- System monitoring alerts, including HA alerts, follow the Telegraf → Victoria Metrics/vmalert → `ns-plug-alert-proxy` path rather than sending legacy portal alerts directly from service scripts. +- System monitoring alerts, including HA alerts, follow the Telegraf → Victoria Metrics/vmalert -> Mimir. - Many local packages are thin wrappers around upstream code. When changing behavior in one of those areas, inspect the matching upstream repo first and treat the local package as integration glue. --- diff --git a/packages/ns-api/Makefile b/packages/ns-api/Makefile index e1579e0aa..db4b2d56a 100644 --- a/packages/ns-api/Makefile +++ b/packages/ns-api/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-api -PKG_VERSION:=3.7.2 +PKG_VERSION:=3.8.0_beta PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-api-$(PKG_VERSION) diff --git a/packages/ns-api/files/ns.backup b/packages/ns-api/files/ns.backup index 3cf1631ab..9dc4fe242 100755 --- a/packages/ns-api/files/ns.backup +++ b/packages/ns-api/files/ns.backup @@ -224,10 +224,9 @@ elif cmd == 'call': elif action == 'registered-delete-backup': try: data = json.load(sys.stdin) - p = subprocess.run(['/usr/sbin/remote-backup', 'delete', data['id']], + subprocess.run(['/usr/sbin/remote-backup', 'delete', data['id']], check=True, capture_output=True, text=True) - # return content - print(p.stdout) + print(json.dumps({'message': 'success'})) except subprocess.CalledProcessError as error: print(json.dumps(utils.generic_error('remote backup delete failed'))) except KeyError as error: diff --git a/packages/ns-api/files/ns.dedalo b/packages/ns-api/files/ns.dedalo index 210257c11..546eb7f53 100755 --- a/packages/ns-api/files/ns.dedalo +++ b/packages/ns-api/files/ns.dedalo @@ -19,6 +19,7 @@ from euci import EUci tmp_dir = "/var/run/" token_file = f"{tmp_dir}/dedalo_token" +pairing_file = f"{tmp_dir}/dedalo_pairing.json" opts = ["network", "hotspot_id", "unit_name", "unit_description", "interface"] ## Utilities @@ -45,17 +46,10 @@ def setup(u): def login(args): u = EUci() try: - p = subprocess.run(['curl', '-L', '--url', f'https://{args["host"]}/api/login', '--header', 'Content-Type: application/json', '--data-binary', json.dumps(args)], check=True, capture_output=True, text=True) + p = subprocess.run(['curl', '-L', '-m', '15', '--connect-timeout', '5', '--url', f'https://{args["host"]}/api/login', '--header', 'Content-Type: application/json', '--data-binary', json.dumps(args)], check=True, capture_output=True, text=True) resp = json.loads(p.stdout) if 'token' in resp: - setup(u) - u.set("dedalo", "config", "splash_page", f'http://{args["host"]}/wings') - u.set("dedalo", "config", "aaa_url", f'https://{args["host"]}/wax/aaa') - u.set("dedalo", "config", "api_url", f'https://{args["host"]}/api') - u.commit("dedalo") - os.makedirs(tmp_dir, exist_ok = True) - with open(token_file, "w") as fp: - fp.write(resp["token"]) + _connect_to_host(u, args["host"], resp["token"]) return {"response": "success"} else: return utils.generic_error("login_failed") @@ -63,6 +57,93 @@ def login(args): print(e, file=sys.stderr) return {"success": False} + +def _connect_to_host(u, host, token, account_name="", account_user=""): + # same side effects as a successful password login: point the unit at + # the chosen hotspot manager and store the session token; the account + # info (from OIDC pairing) is kept to show who the unit is linked to + setup(u) + u.set("dedalo", "config", "splash_page", f'http://{host}/wings') + u.set("dedalo", "config", "aaa_url", f'https://{host}/wax/aaa') + u.set("dedalo", "config", "api_url", f'https://{host}/api') + for opt, value in (("account_name", account_name), ("account_user", account_user)): + if value: + u.set("dedalo", "config", opt, value) + else: + try: + u.delete("dedalo", "config", opt) + except: + pass + u.commit("dedalo") + os.makedirs(tmp_dir, exist_ok = True) + with open(token_file, "w") as fp: + fp.write(token) + +def oidc_start(args): + host = args.get("host") or "my.nethspot.com" + u = EUci() + unit_name = u.get("dedalo", "config", "unit_name", default="") + if not unit_name: + with open('/proc/sys/kernel/hostname', 'r') as fp: + unit_name = fp.read().strip() + try: + p = subprocess.run(['curl', '-s', '-L', '-m', '15', '--connect-timeout', '5', '-X', 'POST', '-w', '\n%{http_code}', '--url', f'https://{host}/api/auth/oidc/device/start', '--header', 'Content-Type: application/json', '--data-binary', json.dumps({"unit_name": unit_name})], check=True, capture_output=True, text=True) + body, _, http_code = p.stdout.rpartition('\n') + # Expected outcomes of the user-provided host (manager without OIDC + # support, wrong/unreachable host) are validation errors: the UI + # shows them inline without the global error toast. + if http_code == '404': + # hotspot manager without OIDC device pairing support + return utils.validation_error("host", "oidc_not_supported") + resp = json.loads(body) + except Exception as e: + print(e, file=sys.stderr) + return utils.validation_error("host", "pairing_start_failed") + if 'device_code' not in resp or 'verification_url' not in resp: + return utils.validation_error("host", "pairing_start_failed") + # the device_code stays on the unit: the browser only ever sees the + # verification_url (carrying the public pair_id) + os.makedirs(tmp_dir, exist_ok = True) + fd = os.open(pairing_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, 'w') as fp: + json.dump({"host": host, "device_code": resp["device_code"]}, fp) + return { + "verification_url": resp["verification_url"], + "expires_in": resp.get("expires_in", 600), + "interval": resp.get("interval", 2), + } + +def oidc_poll(): + u = EUci() + try: + with open(pairing_file, 'r') as fp: + pairing = json.load(fp) + except: + return utils.generic_error("no_pairing_in_progress") + host = pairing["host"] + try: + p = subprocess.run(['curl', '-s', '-L', '-m', '15', '--connect-timeout', '5', '--url', f'https://{host}/api/auth/oidc/device/poll', '--header', 'Content-Type: application/json', '--data-binary', json.dumps({"device_code": pairing["device_code"]})], check=True, capture_output=True, text=True) + resp = json.loads(p.stdout) + except Exception as e: + # transient error talking to the hotspot manager: keep polling + print(e, file=sys.stderr) + return {"status": "pending"} + status = resp.get("status", "") + if status == "ready": + os.remove(pairing_file) + _connect_to_host(u, host, resp["token"], resp.get("account_name", ""), resp.get("logged_by", "")) + return {"status": "success", "account_name": resp.get("account_name", "")} + if status == "failed": + os.remove(pairing_file) + # NB: don't name the key "error" — a top-level "error" key makes + # nethsecurity-api reply 500 (application-error convention) and the + # failed status would never reach the UI as data. + return {"status": "failed", "reason": resp.get("error", "unknown")} + if status == "expired": + os.remove(pairing_file) + return {"status": "expired"} + return {"status": "pending"} + def list_sessions(): process = subprocess.run(["/usr/bin/dedalo", "query", "list"], capture_output=True, text=True) if not process.stdout: @@ -131,7 +212,7 @@ def list_parents(): u = EUci() try: api_url = u.get("dedalo", "config", "api_url") - p = subprocess.run(['curl', '-L', '-s', '--url', f'{api_url}/hotspots', '--header', f"Token: {_get_token()}"], capture_output=True, text=True) + p = subprocess.run(['curl', '-L', '-s', '-m', '15', '--connect-timeout', '5', '--url', f'{api_url}/hotspots', '--header', f"Token: {_get_token()}"], capture_output=True, text=True) resp = json.loads(p.stdout) for p in resp["data"]: parents.append({"id": p["id"], "name": p["name"], "description": p["description"]}) @@ -148,6 +229,12 @@ def unregister(): except Exception as e: print(e, file=sys.stderr) return utils.generic_error("unregister_failed") + try: + u.delete("dedalo", "config", "account_name") + u.delete("dedalo", "config", "account_user") + u.commit("dedalo") + except: + pass try: firewall.delete_linked_sections(EUci(), "dedalo/config") subprocess.run(["/sbin/ifdown", "dedalo"], capture_output=True, check=True) @@ -178,6 +265,10 @@ def get_configuration(): with open('/proc/sys/kernel/hostname', 'r') as fp: ret["unit_name"] = fp.read().strip() ret["connected"] = os.path.exists(token_file) + ret["account_name"] = u.get("dedalo", "config", "account_name", default="") + ret["account_user"] = u.get("dedalo", "config", "account_user", default="") + api_url = u.get("dedalo", "config", "api_url", default="") + ret["manager_host"] = api_url.replace("https://", "").replace("/api", "") return {"configuration": ret} def set_configuration(args): @@ -259,6 +350,8 @@ cmd = sys.argv[1] if cmd == 'list': print(json.dumps({ "login": {"host": "my.nethspot.com", "username": "myuser", "password": "mypassword"}, + "oidc-start": {"host": "my.nethspot.com"}, + "oidc-poll": {}, "list-sessions": {}, "list-parents": {}, "list-devices": {}, @@ -285,6 +378,11 @@ else: elif action == "login": args = json.loads(sys.stdin.read()) ret = login(args) + elif action == "oidc-start": + args = json.loads(sys.stdin.read()) + ret = oidc_start(args) + elif action == "oidc-poll": + ret = oidc_poll() elif action == "set-configuration": args = json.loads(sys.stdin.read()) ret = set_configuration(args) diff --git a/packages/ns-api/files/ns.subscription b/packages/ns-api/files/ns.subscription index dacadeb2c..371ea5f50 100755 --- a/packages/ns-api/files/ns.subscription +++ b/packages/ns-api/files/ns.subscription @@ -11,6 +11,7 @@ import sys import json import subprocess from datetime import datetime +from time import sleep from nethsec import utils from euci import EUci @@ -20,17 +21,19 @@ def register(args): secret = args["secret"] - try: - subprocess.run(["/usr/sbin/register", "enterprise", secret, '5'], check=True, capture_output=True) + enterprise = subprocess.run(["/usr/sbin/register", "enterprise", secret, '5'], capture_output=True) + if enterprise.returncode == 0: return {"result": "success"} - except: - pass - try: - subprocess.run(["/usr/sbin/register", "community", secret, '5'], check=True, capture_output=True) + # Exit code 2: the system is already registered on my. + if enterprise.returncode == 2: + return utils.validation_error("secret", "system_already_registered") + + community = subprocess.run(["/usr/sbin/register", "community", secret, '5'], capture_output=True) + if community.returncode == 0: return {"result": "success"} - except: - return utils.generic_error("invalid_secret_or_server_not_found") + + return utils.generic_error("invalid_secret_or_server_not_found") def unregister(): try: @@ -61,6 +64,22 @@ def info(): type = u.get('ns-plug', 'config', 'type', default='') ret = {"server_id": data["id"], "systemd_id": data["uuid"], "plan": data["subscription"]["subscription_plan"]["name"], "expiration": expiration, "active": active, "type": type} + + if type == "enterprise": + ret["organization"] = data.get("organization", "") + # The system name given on my at creation time (threaded by + # subscription-info from the collect /info payload). + ret["system_name"] = data.get("system_name", "") + ret["plan"] = "Nethesis Enterprise" + ret["system_url"] = f"https://my-proxy-prod.onrender.com/systems/{data['uuid']}" + else: + # Community: link the system to its my.nethserver.com page, matching ns8. + sub_id = (data.get("subscription") or {}).get("id") + if sub_id: + ret["system_url"] = f"https://my.nethserver.com/servers/{sub_id}" + + ret["migrated"] = u.get('ns-plug', 'config', 'migrated', dtype=bool, default=False) + return ret @@ -90,7 +109,8 @@ if cmd == 'list': "unregister": {}, "info": {}, "inventory-status": {}, - "send-inventory": {} + "send-inventory": {}, + "migrate": {}, })) elif cmd == 'call': action = sys.argv[2] @@ -105,5 +125,11 @@ elif cmd == 'call': ret = inventory_status() elif action == "send-inventory": ret = send_inventory() + elif action == "migrate": + try: + subprocess.run(["/usr/libexec/migrate-to-my"], check=True, capture_output=True) + ret = {} + except subprocess.CalledProcessError: + ret = utils.generic_error("failed to migrate") print(json.dumps(ret)) diff --git a/packages/ns-ha/README.md b/packages/ns-ha/README.md index 7ee191088..9b87024d9 100644 --- a/packages/ns-ha/README.md +++ b/packages/ns-ha/README.md @@ -366,8 +366,7 @@ Keepalived Statistics: HA alerts are evaluated by **vmalert** from metrics exported by `/usr/libexec/telegraf-ha-alert`. The collector and HA alert rules are installed by the always-present `telegraf` and -`victoria-metrics` packages. When alerts fire, `ns-plug-alert-proxy` forwards the legacy HA alert IDs -to the monitoring portal if the machine has a valid registration. +`victoria-metrics` packages. Available alerts are: diff --git a/packages/ns-plug/Makefile b/packages/ns-plug/Makefile index 93d50ba8d..4a6c76bce 100644 --- a/packages/ns-plug/Makefile +++ b/packages/ns-plug/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-plug -PKG_VERSION:=1.3.2 +PKG_VERSION:=1.4.0_beta PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-plug-$(PKG_VERSION) @@ -21,7 +21,7 @@ define Package/ns-plug CATEGORY:=NethSecurity TITLE:=NethSecurity controller client URL:=https://github.com/NethServer/nethsecurity-controller/ - DEPENDS:=+openvpn +lscpu +python3-nethsec +python3-yaml +telegraf +victoria-metrics + DEPENDS:=+openvpn +lscpu +python3-nethsec +python3-yaml +telegraf +victoria-metrics +jq PKGARCH:=all endef @@ -44,9 +44,8 @@ if [ -z "$${IPKG_INSTROOT}" ]; then /etc/init.d/cron restart /usr/libexec/ns-plug/40_ns-plug_mwan_hooks /etc/init.d/ns-plug restart - /etc/init.d/ns-plug-alert-proxy enable - /etc/init.d/ns-plug-alert-proxy restart /etc/init.d/vmalert reload + /usr/libexec/migrate-to-my fi exit 0 endef @@ -58,8 +57,6 @@ if [ -z "$${IPKG_INSTROOT}" ]; then crontab -l | grep -v "/usr/sbin/send-inventory" | sort | uniq | crontab - crontab -l | grep -v "/usr/sbin/send-heartbeat" | sort | uniq | crontab - sed -i '/\/usr\/libexec\/ns-plug\/mwan-hooks/d' /etc/mwan3.user - /etc/init.d/ns-plug-alert-proxy stop - /etc/init.d/ns-plug-alert-proxy disable fi exit 0 endef @@ -77,11 +74,10 @@ define Package/ns-plug/install $(INSTALL_DIR) $(1)/usr/libexec/ns-plug $(INSTALL_DIR) $(1)/usr/libexec/mwan-hooks $(INSTALL_BIN) ./files/ns-plug.init $(1)/etc/init.d/ns-plug - $(INSTALL_BIN) ./files/ns-plug-alert-proxy.init $(1)/etc/init.d/ns-plug-alert-proxy $(INSTALL_BIN) ./files/ns-plug $(1)/usr/sbin/ns-plug - $(INSTALL_BIN) ./files/ns-plug-alert-proxy $(1)/usr/sbin/ns-plug-alert-proxy $(INSTALL_BIN) ./files/distfeed-setup $(1)/usr/sbin/distfeed-setup $(INSTALL_BIN) ./files/apk-official $(1)/usr/sbin/apk-official + $(INSTALL_BIN) ./files/migrate-to-my $(1)/usr/libexec $(INSTALL_BIN) ./files/remote-backup $(1)/usr/sbin $(INSTALL_BIN) ./files/send-backup $(1)/usr/sbin $(INSTALL_BIN) ./files/send-heartbeat $(1)/usr/sbin diff --git a/packages/ns-plug/files/config b/packages/ns-plug/files/config index 9f32f0262..38537c6ab 100644 --- a/packages/ns-plug/files/config +++ b/packages/ns-plug/files/config @@ -5,6 +5,8 @@ config main 'config' option unit_name '' option tls_verify '1' option backup_url 'https://backupd.nethesis.it' + option collect_url 'https://my-proxy-prod.onrender.com/collect/api/systems' + option notifier_url '' option repository_url 'https://updates.nethsecurity.nethserver.org' option channel '' option tun_mtu '' diff --git a/packages/ns-plug/files/migrate-to-my b/packages/ns-plug/files/migrate-to-my new file mode 100644 index 000000000..ccf00532d --- /dev/null +++ b/packages/ns-plug/files/migrate-to-my @@ -0,0 +1,103 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# Idempotent one-shot migration from legacy my.nethesis.it / backupd +# credentials to the my collect native credentials, so the unit can +# authenticate directly against the my collect endpoints. +# +# Only enterprise units are migrated. Community (my.nethserver.com) +# has its own infrastructure and keeps using the legacy send-* +# endpoints — no credential rotation is applicable there. +# +# ns-plug.config.migrated='1' is the persistent marker. It is written +# by this script after a successful rotation, and also by +# /usr/sbin/register when it registers a fresh unit directly against +# the my collect endpoint (so a brand new install never triggers the +# rotation path and never hits /proxy/credentials with unmapped +# credentials). +# +# On the first successful invocation the script: +# 1. Calls the my translation proxy's /proxy/credentials endpoint +# with the legacy Basic-Auth pair and retrieves the mapped my +# system key / secret. +# 2. Writes the new credentials to ns-plug.config.system_id / secret +# and preserves the legacy pair under legacy_system_id / +# legacy_secret (for audit and manual rollback). +# 3. Re-asserts ns-plug.config.collect_url, because /etc/config/ +# ns-plug is a conffile: on registered units opkg keeps the +# user-modified copy across upgrades, so a new default alone +# would not reach them. +# 4. Sets the migrated='1' marker. +# +# The uci commit is atomic — a partial write cannot leave the unit in +# an inconsistent half-migrated state. +# + +# Marker: set only after a successful rotation or a native my register. +[ "$(uci -q get ns-plug.config.migrated)" = "1" ] && exit 0 + +# Community units stay on the legacy my.nethserver.com infrastructure. +if [ "$(uci -q get ns-plug.config.type)" != "enterprise" ]; then + exit 0 +fi + +SYSTEM_ID=$(uci -q get ns-plug.config.system_id) +SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) +if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then + # Unregistered unit — nothing to migrate yet. + exit 0 +fi + +# Fetch the mapped my credentials via the translation proxy. +resp=$(/usr/bin/curl --silent --location-trusted --fail-with-body \ + --max-time 30 --retry 2 \ + --user "$SYSTEM_ID:$SYSTEM_SECRET" \ + https://my-proxy-prod.onrender.com/proxy/credentials 2>/dev/null) || { + echo "credential fetch failed; will retry on next run" + exit 1 +} + +new_key=$(echo "$resp" | jq -r '.data.system_key // empty' 2>/dev/null) +new_secret=$(echo "$resp" | jq -r '.data.system_secret // empty' 2>/dev/null) +if [ -z "$new_key" ] || [ -z "$new_secret" ]; then + echo "credentials missing in response" + exit 2 +fi + +# Rotate atomically; legacy pair preserved for audit / rollback. +uci -q batch </dev/null || true + +# The enterprise feeds hold the credentials inside their URLs, written at +# registration: rotating the pair here would leave them authenticating as the +# legacy system, which works only until the old my goes away. Re-run just the +# feed hooks -- the other register hooks act on a first registration. +for feed_setup in /usr/sbin/ts-ip /usr/sbin/ts-dns /usr/sbin/distfeed-setup; do + [ -x "$feed_setup" ] && "$feed_setup" || true +done +/etc/init.d/banip reload 2>/dev/null || true +/etc/init.d/adblock reload 2>/dev/null || true + +echo "migrated to my collect credentials" +exit 0 diff --git a/packages/ns-plug/files/ns-plug-alert-proxy b/packages/ns-plug/files/ns-plug-alert-proxy deleted file mode 100644 index b1167c7b3..000000000 --- a/packages/ns-plug/files/ns-plug-alert-proxy +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/python3 - -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -""" -Alert proxy: receives Alertmanager-like notifications from vmalert and -forwards selected alerts to the legacy my.nethesis.it / my.nethserver.com -monitoring portals. - -Only the following alerts are forwarded: - - WanDown → wan::down - - DiskSpaceCritical → df:root:percent_bytes:free (path=/) - df:boot:percent_bytes:free (path=/boot) - - BackupEncryptionDisabled → backup:config:notencrypted - - StorageStatus → storage:status - - HaPrimaryFailed → ha:primary:failed - - HaSyncFailed → ha:sync:failed - -All other alerts are silently dropped. -If the machine is not registered (no system_id/secret in UCI), all alerts -are silently dropped. - -Firing/resolved state is determined from the Alertmanager-standard endsAt -field: if endsAt is in the future (or zero/missing) the alert is FAILURE; -if endsAt is in the past the alert is OK. -""" - -import json -import logging -import re -import sys -import time -import urllib.request -from datetime import datetime, timezone -from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn -from euci import EUci - -LISTEN_ADDR = "127.0.0.1" -LISTEN_PORT = 9095 - -_DISK_PATH_MAP = { - "/": "df:root:percent_bytes:free", - "/boot": "df:boot:percent_bytes:free", -} - -_ZERO_TIME = "0001-01-01T00:00:00Z" -# vmalert uses nanosecond precision; strip to microseconds for Python parsing -_NANO_RE = re.compile(r"(\.\d{6})\d+(Z|[+-]\d{2}:\d{2})$") - - -def _is_firing(alert): - """Return True if the alert is currently firing based on endsAt.""" - ends_at_str = alert.get("endsAt", "") - if not ends_at_str or ends_at_str == _ZERO_TIME: - return True - ends_at_str = _NANO_RE.sub(r"\1\2", ends_at_str) - ends_at_str = ends_at_str.replace("Z", "+00:00") - try: - ends_at = datetime.fromisoformat(ends_at_str) - return ends_at > datetime.now(timezone.utc) - except Exception: - return True - - -def _map_alert_id(alert_name, labels): - """Return the legacy alert_id string, or None if the alert is not mapped.""" - if alert_name == "WanDown": - iface = labels.get("interface", "unknown") - return f"wan:{iface}:down" - if alert_name == "DiskSpaceCritical": - path = labels.get("path", "") - return _DISK_PATH_MAP.get(path) - if alert_name == "BackupEncryptionDisabled": - return "backup:config:notencrypted" - if alert_name == "StorageStatus": - return "storage:status" - if alert_name == "HaPrimaryFailed": - return "ha:primary:failed" - if alert_name == "HaSyncFailed": - return "ha:sync:failed" - return None - - -def _send_alert(system_id, secret, alerts_url, alert_id, status, retry=3): - url = alerts_url.rstrip("/") + "/alerts/store" - payload = json.dumps( - {"lk": system_id, "alert_id": alert_id, "status": status} - ).encode() - req = urllib.request.Request( - url, - data=payload, - method="POST", - headers={ - "Authorization": f"token {secret}", - "Content-Type": "application/json", - "Accept": "application/json", - }, - ) - try: - with urllib.request.urlopen(req, timeout=60) as resp: - logging.debug(f"Alert sent: {alert_id} {status} → {resp.status}") - except Exception as ex: - if retry > 0: - logging.warning(f"Alert send failed: {alert_id} {ex} — retrying in 20s") - time.sleep(20) - _send_alert(system_id, secret, alerts_url, alert_id, status, retry - 1) - else: - logging.warning(f"Alert send aborted: {alert_id} {ex}") - - -class _AlertHandler(BaseHTTPRequestHandler): - def log_message(self, format, *args): - # Suppress access log - pass - - def do_GET(self): - self.send_response(200) - self.end_headers() - - def do_POST(self): - if self.system_id is None or self.secret is None or self.alerts_url is None: - logging.debug("Alert dropped (not registered): no system_id or secret") - self.send_response(200) - self.end_headers() - return - try: - length = int(self.headers.get("Content-Length", 0)) - body = self.rfile.read(length) - data = json.loads(body) - except Exception as ex: - self.send_response(400) - self.end_headers() - self.wfile.write(str(ex).encode()) - return - - if type(data) is list: - alerts = data - else: - alerts = data.get("alerts", []) - for alert in alerts: - labels = alert.get("labels", {}) - alert_name = labels.get("alertname", "") - legacy_status = "FAILURE" if _is_firing(alert) else "OK" - - alert_id = _map_alert_id(alert_name, labels) - if not alert_id: - logging.debug(f"Alert dropped (no mapping): {alert_name} {labels}") - continue - - _send_alert(self.system_id, self.secret, self.alerts_url, alert_id, legacy_status) - - self.send_response(200) - self.end_headers() - - def __init__(self, *args, **kwargs): - uci = EUci() - self.system_id = uci.get("ns-plug", "config", "system_id", default=None) - self.secret = uci.get("ns-plug", "config", "secret", default=None) - self.alerts_url = uci.get("ns-plug", "config", "alerts_url", default=None) - super().__init__(*args, **kwargs) - - -class _ThreadingHTTPServer(ThreadingMixIn, HTTPServer): - daemon_threads = True - - -def main(): - uci = EUci() - loglevel_str = uci.get("ns-plug", "config", "alert_proxy_loglevel", default="warning") - loglevel = getattr(logging, loglevel_str.upper(), logging.WARNING) - logging.basicConfig(level=loglevel, format="%(message)s", stream=sys.stderr) - server = _ThreadingHTTPServer((LISTEN_ADDR, LISTEN_PORT), _AlertHandler) - logging.info(f"alert-proxy listening on {LISTEN_ADDR}:{LISTEN_PORT}") - server.serve_forever() - - -if __name__ == "__main__": - main() diff --git a/packages/ns-plug/files/ns-plug-alert-proxy.init b/packages/ns-plug/files/ns-plug-alert-proxy.init deleted file mode 100644 index 38ce01012..000000000 --- a/packages/ns-plug/files/ns-plug-alert-proxy.init +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh /etc/rc.common - -# -# Copyright (C) 2026 Nethesis S.r.l. -# SPDX-License-Identifier: GPL-2.0-only -# - -START=95 -STOP=4 -USE_PROCD=1 - -start_service() { - procd_open_instance - procd_set_param stdout 1 - procd_set_param stderr 1 - procd_set_param command '/usr/sbin/ns-plug-alert-proxy' - procd_set_param respawn 3600 5 0 - procd_close_instance -} - -service_triggers() -{ - procd_add_reload_trigger "ns-plug" -} - -reload_service() -{ - stop - start -} diff --git a/packages/ns-plug/files/register b/packages/ns-plug/files/register index 77d8b3de6..5b7871bfb 100755 --- a/packages/ns-plug/files/register +++ b/packages/ns-plug/files/register @@ -42,11 +42,21 @@ case "$type" in "${url}machine/info" | jq -r ".uuid" 2>/dev/null) ;; enterprise) - url="https://my.nethesis.it/api/" + url="https://my-proxy-prod.onrender.com/backend/api/" - system_id=$(curl -s -m $timeout --retry 3 -L \ + register_resp=$(curl -s -m $timeout --retry 3 -L -w '\n%{http_code}' \ -H "Content-Type: application/json" -H "Accept: application/json" \ - -d '{"secret": "'$secret'"}' "${url}systems/info" | jq -r ".uuid" 2>/dev/null) + -d '{"system_secret": "'$secret'"}' "${url}systems/register") + http_code=$(echo "$register_resp" | sed -n '$p') + register_resp=$(echo "$register_resp" | sed '$d') + + # A system key is one-shot. + if [ "$http_code" = "409" ]; then + >&2 echo "[ERROR] system already registered on my" + exit 2 + fi + + system_id=$(echo "$register_resp" | jq -r '.data.system_key // empty' 2>/dev/null) ;; *) exit_error "Invalid type '$type'" @@ -68,9 +78,10 @@ case "$type" in ;; enterprise) uci set ns-plug.config.type="enterprise" - uci set ns-plug.config.alerts_url="https://my.nethesis.it/isa/" uci set ns-plug.config.api_url="$url" - uci set ns-plug.config.inventory_url="https://my.nethesis.it/isa/inventory/store/" + uci set ns-plug.config.collect_url="https://my-proxy-prod.onrender.com/collect/api/systems" + uci set ns-plug.config.notifier_url=https://my-proxy-prod.onrender.com/collect/api/services/mimir/alertmanager + uci set ns-plug.config.migrated="1" ;; esac @@ -79,9 +90,8 @@ uci set ns-plug.config.secret="$secret" uci set ns-plug.config.repository_url="https://$system_id:$secret@distfeed.nethesis.it/repository/$type/nethsecurity" uci commit ns-plug reload_config -# Register the machine by sending the inventory for the first time -send-inventory send-heartbeat +send-inventory exit_code=$? # Execute register hooks diff --git a/packages/ns-plug/files/remote-backup b/packages/ns-plug/files/remote-backup index 9a12d0ef4..e588fd80b 100755 --- a/packages/ns-plug/files/remote-backup +++ b/packages/ns-plug/files/remote-backup @@ -9,33 +9,78 @@ # Manage remote backup # +set -o pipefail + function exit_error { >&2 echo "[ERROR] $@" exit 1 } function help { - >&2 echo "Usage: $0 " + >&2 echo "Usage: $0 " >&2 echo "Commands:" - >&2 echo " - list: retrieve the list of available backups from remote server" - >&2 echo " - download [output]: download the given backup, if 'output' is empty downloaded file will be named as as 'file'" - >&2 echo " - upload : upload the given backup" + >&2 echo " - list: fetch the list of backups stored for this system" + >&2 echo " - download [output]: download the backup ; defaults to writing to a file named " + >&2 echo " - upload : upload a backup file" + >&2 echo " - delete : remove the backup " } SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) TYPE=$(uci -q get ns-plug.config.type) -URL=$(uci -q get ns-plug.config.backup_url) -if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ] || [ -z "$URL" ]; then - exit_error "System ID, system secret or backup url not found. Please configure ns-plug." +if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then + exit_error "System ID or system secret not found. Please configure ns-plug." +fi + +cmd=${1:-list} + +if [ "$TYPE" = "enterprise" ]; then + BASE="$(uci -q get ns-plug.config.collect_url)/backups" + curl_args="--silent --show-error --location-trusted --fail-with-body --user $SYSTEM_ID:$SYSTEM_SECRET" + + case "$cmd" in + list) + curl $curl_args "$BASE" + ;; + download) + file=$2 + [ -z "$file" ] && exit_error "No file specified" + output=${3-$file} + curl $curl_args -o "$output" "$BASE/$file" + ;; + upload) + file=$2 + [ -z "$file" ] && exit_error "No file specified" + curl $curl_args -X POST \ + -H "Content-Type: application/octet-stream" \ + -H "X-Filename: $(basename "$file")" \ + --data-binary "@$file" \ + "$BASE" + ;; + delete) + file=$2 + [ -z "$file" ] && exit_error "No file specified" + curl $curl_args -X DELETE "$BASE/$file" + ;; + *) + help + ;; + esac + + exit $? +fi + +# Community (legacy): backupd.nethesis.it with the /$TYPE/api/v2/backup/ +# URL layout. Unchanged from the pre-migration behaviour. +URL=$(uci -q get ns-plug.config.backup_url) +if [ -z "$URL" ]; then + exit_error "Backup URL not set. Please configure ns-plug." fi curl_args="--silent --location-trusted --user $SYSTEM_ID:$SYSTEM_SECRET" base_url="$URL/$TYPE/api/v2/backup/" -cmd=${1:-list} - case "$cmd" in list) curl $curl_args $base_url @@ -47,36 +92,23 @@ case "$cmd" in fi output=${3-$file} curl $curl_args $base_url$file -J -o "$output" - ;; - upload) - file=$2 - if [ -z "$file" ]; then + ;; + upload) + file=$2 + if [ -z "$file" ]; then exit_error "No file specified" fi - curl $curl_args $base_url --upload-file $file - rc=$? - # Temporary dual-send to new my.nethesis.it via the translation - # proxy, same pattern used by send-heartbeat / send-inventory. - # To be removed once the migration is complete. - if [ "$TYPE" = "enterprise" ]; then - # Strip the directory path first to get just the filename - filename="${file##*/}" - curl $curl_args -X POST \ - -H "Content-Type: application/octet-stream" \ - -H "X-Filename: ${filename}" \ - --data-binary "@$file" https://my.nethesis.it/proxy/backup >/dev/null || : - fi - exit $rc - ;; - delete) - file=$2 - if [ -z "$file" ]; then + curl $curl_args $base_url --upload-file $file + ;; + delete) + file=$2 + if [ -z "$file" ]; then exit_error "No file specified" fi - curl $curl_args -X DELETE $base_url$file - ;; + curl $curl_args -X DELETE $base_url$file + ;; - *) - help - ;; + *) + help + ;; esac diff --git a/packages/ns-plug/files/send-backup b/packages/ns-plug/files/send-backup index 707fdc439..8aa52057b 100644 --- a/packages/ns-plug/files/send-backup +++ b/packages/ns-plug/files/send-backup @@ -29,7 +29,11 @@ send() { if [ -s "$PASSPHRASE" ]; then # send encrypted backup gpg --batch -c --yes --passphrase-file "$PASSPHRASE" "$BACKUP" - remote-backup upload "$BACKUP.gpg" + # to surface error log from cron + if ! err=$(remote-backup upload "$BACKUP.gpg" 2>&1); then + logger -t send-backup "backup upload failed: $err" + exit 1 + fi mv "$MD5" "$MD5_LAST" else # password not set, abort upload diff --git a/packages/ns-plug/files/send-heartbeat b/packages/ns-plug/files/send-heartbeat index 36b328349..15eef50ea 100755 --- a/packages/ns-plug/files/send-heartbeat +++ b/packages/ns-plug/files/send-heartbeat @@ -9,7 +9,6 @@ SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) -URL=$(uci -q get ns-plug.config.alerts_url)"heartbeats/store" TYPE=$(uci -q get ns-plug.config.type) if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then @@ -17,14 +16,18 @@ if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then exit 0 fi -/usr/bin/curl -m 180 --retry 3 -L -s \ - --header "Authorization: token $SYSTEM_SECRET" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-raw '{"lk": "'$SYSTEM_ID'"}' "$URL" >/dev/null - -# Temporary send data to new endpoint -# To be removed when the migration to new my.nethesis.it will be completed -if [ "$TYPE" = "enterprise" ]; then - /usr/bin/curl -m 180 --retry 3 -L -s -X POST \ - --user "$SYSTEM_ID:$SYSTEM_SECRET" https://my.nethesis.it/proxy/heartbeat >/dev/null - exit 0 -fi +case "$TYPE" in + enterprise) + /usr/bin/curl -m 30 --retry 3 -L -sSf -X POST \ + --user "$SYSTEM_ID:$SYSTEM_SECRET" \ + "$(uci -q get ns-plug.config.collect_url)/heartbeat" >/dev/null + ;; + community) + URL=$(uci -q get ns-plug.config.alerts_url)"heartbeats/store" + /usr/bin/curl -m 180 --retry 3 -L -s \ + --header "Authorization: token $SYSTEM_SECRET" \ + --header "Content-Type: application/json" \ + --header "Accept: application/json" \ + --data-raw '{"lk": "'$SYSTEM_ID'"}' "$URL" >/dev/null + ;; +esac diff --git a/packages/ns-plug/files/send-inventory b/packages/ns-plug/files/send-inventory index a670e9925..ece3a8398 100755 --- a/packages/ns-plug/files/send-inventory +++ b/packages/ns-plug/files/send-inventory @@ -9,7 +9,6 @@ SYSTEM_ID=$(uci -q get ns-plug.config.system_id) SYSTEM_SECRET=$(uci -q get ns-plug.config.secret) -URL=$(uci -q get ns-plug.config.inventory_url) TYPE=$(uci -q get ns-plug.config.type) if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then @@ -17,28 +16,28 @@ if [ -z "$SYSTEM_ID" ] || [ -z "$SYSTEM_SECRET" ]; then exit 0 fi -echo "{\"data\": {\"lk\": \"$SYSTEM_ID\", \"data\": $(/usr/sbin/inventory) }}" | \ -/usr/bin/curl -m 180 --retry 5 -L -s \ - --header "Authorization: token $SYSTEM_SECRET" --header "Content-Type: application/json" --header "Accept: application/json" \ - --data-binary @- "$URL" > /dev/null - -if [ $? -ne 0 ]; then - status="error" -else +status="error" + +case "$TYPE" in + enterprise) + /usr/sbin/phonehome | /usr/bin/curl -m 180 --retry 3 -L -sSf -X POST \ + --user "$SYSTEM_ID:$SYSTEM_SECRET" \ + -H "Content-Type: application/json" \ + --data-binary @- \ + "$(uci -q get ns-plug.config.collect_url)/inventory" > /dev/null + ;; + community) + echo "{\"data\": {\"lk\": \"$SYSTEM_ID\", \"data\": $(/usr/sbin/inventory) }}" | \ + /usr/bin/curl -m 180 --retry 5 -L -s \ + --header "Authorization: token $SYSTEM_SECRET" \ + --header "Content-Type: application/json" \ + --header "Accept: application/json" \ + --data-binary @- "$(uci -q get ns-plug.config.inventory_url)" > /dev/null + ;; +esac + +if [ $? -eq 0 ]; then status="success" fi echo '{"status": "'$status'", "last_attempt": "'$(date -Iseconds)'"}' > /tmp/inventory-sent.json - -if [ "$TYPE" = "enterprise" ]; then - # Update registration date - /usr/bin/curl -m 180 --retry 5 -L -s \ - --header "Content-Type: application/json" --header "Accept: application/json" \ - -d '{"secret":"'$SYSTEM_SECRET'"}' https://my.nethesis.it/api/systems/info >/dev/null - - # Temporary send data to new endpoint - # To be removed when the migration to new my.nethesis.it will be completed - /usr/sbin/phonehome | /usr/bin/curl -m 180 --retry 3 -L -s --user "$SYSTEM_ID:$SYSTEM_SECRET" \ - -H "Content-Type: application/json" \ - --data-binary @- https://my.nethesis.it/proxy/inventory >/dev/null || : -fi diff --git a/packages/ns-plug/files/subscription-info b/packages/ns-plug/files/subscription-info index d78d3cbdd..8044984dd 100755 --- a/packages/ns-plug/files/subscription-info +++ b/packages/ns-plug/files/subscription-info @@ -22,13 +22,33 @@ fi type=$(uci -q get ns-plug.config.type) secret=$(uci -q get ns-plug.config.secret) -url=$(uci -q get ns-plug.config.api_url | sed 's/\/$//') if [ "$type" = "enterprise" ]; then - curl -f -s -m $timeout --retry-delay 1 --retry 2 -L \ - -H "Content-Type: application/json" -H "Accept: application/json" \ - -d '{"secret": "'$secret'"}' "$url/systems/info" + resp=$(/usr/bin/curl -f -s -m $timeout --retry-delay 1 --retry 2 -L \ + -H "Accept: application/json" \ + --user "$system_id:$secret" \ + "$(uci -q get ns-plug.config.collect_url)/info") || exit $? + + jq -c ' + .data as $s | + { + uuid: ($s.system_id // ""), + id: ($s.system_key // ""), + system_name: ($s.name // ""), + organization: ($s.organization.name // ""), + subscription: { + status: (if $s.registered and (($s.suspended // false) | not) then "valid" else "invalid" end), + valid_until: null, + subscription_plan: { name: ($s.organization.name // "-") } + } + } + ' </dev/null +# Release the legacy slot on my-old for migrated enterprise units +if [ "$(uci -q get ns-plug.config.type)" = "enterprise" ]; then + LEGACY_ID=$(uci -q get ns-plug.config.legacy_system_id) + LEGACY_SECRET=$(uci -q get ns-plug.config.legacy_secret) + if [ -n "$LEGACY_ID" ] && [ -n "$LEGACY_SECRET" ]; then + curl -s -m 180 --retry 3 -L \ + -H "Content-type: application/json" -H "Accept: application/json" \ + -d "{\"lk\":\"$LEGACY_ID\",\"secret\":\"$LEGACY_SECRET\"}" \ + https://my-proxy-prod.onrender.com/api/Utils/freekey >/dev/null + fi +fi # Reset ns-plug configuration uci set ns-plug.config.type="" -uci set ns-plug.config.alerts_url="" -uci set ns-plug.config.api_url="" -uci set ns-plug.config.inventory_url="" uci set ns-plug.config.system_id="" uci set ns-plug.config.secret="" uci set ns-plug.config.repository_url="https://updates.nethsecurity.nethserver.org/$(cat /etc/repo-channel)" +uci set ns-plug.config.notifier_url="" +uci -q delete ns-plug.config.collect_url +uci -q delete ns-plug.config.migrated +uci -q delete ns-plug.config.legacy_system_id +uci -q delete ns-plug.config.legacy_secret # Save config uci commit ns-plug diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index d0b116059..6c964c0e4 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-threat_shield -PKG_VERSION:=1.0.2 +PKG_VERSION:=1.0.3_beta PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-threat_shield-$(PKG_VERSION) diff --git a/packages/ns-threat_shield/files/ts-ip b/packages/ns-threat_shield/files/ts-ip index 8acc85333..bd4fd7f36 100755 --- a/packages/ns-threat_shield/files/ts-ip +++ b/packages/ns-threat_shield/files/ts-ip @@ -33,8 +33,17 @@ TYPE=$(uci -q get ns-plug.config.type) if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then jq -s '.[0] * .[1]' /etc/banip/banip.nethesis.feeds /etc/banip/banip.feeds \ | sed -e "s/__USER__/$SYSTEM_ID/" -e "s/__PASSWORD__/$SYSTEM_SECRET/" -e "s/__TYPE__/$TYPE/" > /etc/banip/banip.custom.feeds - if ! uci -q get banip.global.ban_allowurl | grep -q bl.nethesis.it; then - uci add_list banip.global.ban_allowurl="https://$SYSTEM_ID:$SYSTEM_SECRET@bl.nethesis.it/plain/$TYPE/nethesis-blacklists/whitelist.global" + # The credentials are part of the URL, so an entry written before a key + # rotation keeps fetching as the old pair. Enforce one entry, the current + # one: compare the whole set, and del_list per entry (a multi-line value + # matches nothing). + allow_current="https://$SYSTEM_ID:$SYSTEM_SECRET@bl.nethesis.it/plain/$TYPE/nethesis-blacklists/whitelist.global" + allow_have=$(uci -q get banip.global.ban_allowurl | tr " " "\n" | grep bl.nethesis.it) + if [ "$allow_have" != "$allow_current" ]; then + for allow_stale in $allow_have; do + uci del_list banip.global.ban_allowurl="$allow_stale" + done + uci add_list banip.global.ban_allowurl="$allow_current" uci commit banip fi else diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 5403c6c47..852668498 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -7,12 +7,17 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-ui # renovate: datasource=github-releases depName=NethServer/nethsecurity-ui -PKG_VERSION:=2.23.4 +PKG_VERSION:=2.24.0_beta PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) +# TEMP (my cutover): pinned to nethsecurity-ui#746 (feat/backup-my-api) so the image +# bundles the my-native backup UI alongside this appliance cutover. The UI reads the +# new collect backup payload; on a device still on backupd it would break, hence it must +# ship together with this PR. BEFORE MERGING: revert to PKG_SOURCE_VERSION:=$(PKG_VERSION) +# and bump PKG_VERSION to the nethsecurity-ui release that carries #746. +PKG_SOURCE_VERSION:=6ef65208782728b320421f6e6d2bceea1b1da611 PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile index c8facc1db..827999560 100644 --- a/packages/telegraf/Makefile +++ b/packages/telegraf/Makefile @@ -8,7 +8,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=telegraf # renovate: datasource=github-tags depName=influxdata/telegraf PKG_VERSION:=1.39.1 -PKG_RELEASE:=3 +PKG_RELEASE:=4 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/influxdata/telegraf/tar.gz/v$(PKG_VERSION)? diff --git a/packages/telegraf/README.md b/packages/telegraf/README.md index b6cb55a73..70284b418 100644 --- a/packages/telegraf/README.md +++ b/packages/telegraf/README.md @@ -65,7 +65,6 @@ ns-clm ns-flashstart ns-flows ns-plug -ns-plug-alert-proxy ns-stats ns-ui odhcpd diff --git a/packages/telegraf/files/telegraf-services b/packages/telegraf/files/telegraf-services index 1eff7a2e9..1bcf41391 100644 --- a/packages/telegraf/files/telegraf-services +++ b/packages/telegraf/files/telegraf-services @@ -22,6 +22,7 @@ import json import subprocess import sys + MONITORED_SERVICES = { "conntrackd", "cron", @@ -38,7 +39,6 @@ MONITORED_SERVICES = { "ns-flashstart", "ns-flows", "ns-plug", - "ns-plug-alert-proxy", "ns-stats", "ns-ui", "odhcpd", @@ -56,6 +56,7 @@ MONITORED_SERVICES = { # Excluded service: adblock + def get_service_list(): result = subprocess.run( ["ubus", "call", "service", "list"], diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile index 77d9fc84a..15de80400 100644 --- a/packages/victoria-metrics/Makefile +++ b/packages/victoria-metrics/Makefile @@ -8,7 +8,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=victoria-metrics # renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics PKG_VERSION:=1.146.0 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaMetrics/tar.gz/v$(PKG_VERSION)? diff --git a/packages/victoria-metrics/README.md b/packages/victoria-metrics/README.md index 6dcc24bff..a32894a03 100644 --- a/packages/victoria-metrics/README.md +++ b/packages/victoria-metrics/README.md @@ -46,6 +46,21 @@ config victoriametrics 'main' - `storage_path`: Where to store metrics data (default: `/var/lib/victoriametrics`, auto-detects `/mnt/data/victoriametrics` if available) - `retention_period`: How long to keep metrics (`1d`, `7d`, `30d`, `1y`, etc.) (default: `7d`, auto-detects `1y` if not set) +### Advanced: Extra vmalert Flags + +`/etc/config/vmalert`'s `main` section accepts an `additional_parameters` list: extra `vmalert` +CLI flags, one per entry, appended after every other flag on the command line. + +For example, to point vmalert at an external Alertmanager notifier: + +```bash +uci add_list vmalert.main.additional_parameters='-notifier.url=' +uci commit vmalert +reload_config +``` + +See the [vmalert documentation](https://docs.victoriametrics.com/vmalert/) for the full flag list. + ### Accessing the Web UI By default the server is accessible only on localhost for security. @@ -137,72 +152,16 @@ The engine processes an active incident through three phases: Because real-world server incidents do not align perfectly with the monitoring engine's internal execution clock, notifications feature a variable delay window of 5 to 10 minutes from the actual start of the incident. -## Forwarding alerts to my.nethesis.it - -[my](https://github.com/NethServer/my/) uses Grafana Mimir as a multi-tenant -alertmanager for cloud-side alert processing. Enterprise systems forward their -alerts to it automatically, mirroring `send-heartbeat` / `send-inventory`: -vmalert POSTs alerts to the credential-translation proxy at -`https://my.nethesis.it/proxy/alerts` using the ns-plug credentials -(`system_id` / `secret`), which the proxy maps to the new my credentials before -forwarding them to the Mimir alertmanager. No manual configuration is needed — -it is enabled whenever `ns-plug.config.type` is `enterprise` and the system is -registered (`system_id` / `secret` set). vmalert always also notifies the local -ns-plug-alert-proxy (`http://127.0.0.1:9095`), which handles the legacy path and -unregistered machines. - -By default, ns-plug-alert proxy logs only when an alert can't be forwarded to legacy my.nethesis.it. -To increase verbosity and debug all communications with the portal, -set `ns-plug.config.alert_proxy_loglevel` to `info` or `debug` and restart ns-plug-alert-proxy: -```bash -uci set ns-plug.config.alert_proxy_loglevel='debug' -uci commit ns-plug -/etc/init.d/ns-plug-alert-proxy restart -``` - -> Migration note: the my switch-off release will repoint this from -> `/proxy/alerts` to the native collect endpoint -> (`/collect/api/services/mimir/alertmanager`) with rotated credentials. - ## Alert notifications System alerts are handled by vmalert (Victoria Metrics alert evaluation engine) which evaluates alert rules against metrics collected by telegraf. -When a rule transitions from `Pending` to `Firing`, vmalert sends an Alertmanager notification to the following endponts: -- ns-plug-alert-proxy, listening on port 9095, which forwards only some alerts to the legacy monitoring portal -- https://my.nethesis.it/proxy/alerts, wich forwards all alerts to the new Mimir alertmanager +When a rule transitions from `Pending` to `Firing`, vmalert sends to remote Mimir instance if active subscription is present. vmalert sends a notification for firing alerts every `interval`, set to 5 minutes for most alerts, until the alert resolves. When the alert resolves, vmalert sends 4 notifications at 5-minute intervals to ensure the resolution is received by the alertmanager (or the proxy) even if the first notification is lost. -**Migration note** - -When legacy my.nethesis.it will be replaced with the new one: -- remove ns-plug-alert-proxy from the system (caveat: also my.nethserver.com will not receive alerts anymore) -- change vmalert configuration to send alerts directly to the new Mimir alertmanager endpoint: replace `/proxy/alerts` - with the native collect endpoint `/collect/api/services/mimir/alertmanager` with rotated credentials. - -### ns-plug-alert-proxy - -The proxy forwards only the following legacy alerts: -| Alert | Condition | Legacy alert_id | -|---|---|---| -| `WanDown` | WAN interface offline for 2m | `wan::down` | -| `DiskSpaceCritical` | Disk usage > 90% for 2m | `df:root:percent_bytes:free` or `df:boot:percent_bytes:free` | -| `StorageStatus` | Storage status is error | `storage:status` | -| `HaPrimaryFailed` | Backup node became master | `ha:primary:failed` | -| `HaSyncFailed` | HA sync failure detected on the primary node | `ha:sync:failed` | - -All other alert are silently dropped by the proxy. -If the machine does not have a subscription, all alerts are silently dropped. - -The proxy starts automatically at boot regardless of registration state. -By default, firing/resolved state is determined from the Alertmanager-standard `endsAt` field: -if `endsAt` is in the future (or zero/missing) a **FAILURE** is sent; if `endsAt` is in -the past an **OK** is sent. HA recovery/failover event alerts override this default mapping so -they can keep the legacy `ha:primary:failed` semantics. - ## Alert history The `vmalert` alerts keeps the state of all active alerts inside VictoriaMetrics using the remote-write protocol. diff --git a/packages/victoria-metrics/files/vmalert.initd b/packages/victoria-metrics/files/vmalert.initd index 3abf258f9..18d9bd382 100644 --- a/packages/victoria-metrics/files/vmalert.initd +++ b/packages/victoria-metrics/files/vmalert.initd @@ -12,6 +12,10 @@ USE_PROCD=1 PROG="/usr/bin/vmalert" RULE_DIR="/etc/vmalert/rules" +function append_params() { + procd_append_param command $@ +} + start_service() { config_load vmalert 2>/dev/null || true @@ -19,26 +23,14 @@ start_service() { config_get datasource_url main datasource_url "http://localhost:8428" config_get http_listen_addr main http_listen_addr "127.0.0.1:8081" - # Forward alerts to the new my.nethesis.it during the migration window, - # mirroring send-heartbeat / send-inventory: enterprise systems POST to the - # credential-translation proxy at my.nethesis.it/proxy/alerts using the - # ns-plug credentials (system_id:secret), which the proxy maps to the new my - # credentials. vmalert appends /api/v2/alerts to the notifier URL. The my - # switch-off release will repoint this to the native collect path. - local system_id system_secret system_type alerts_disabled notifier_url notifier_user notifier_pass + # Forward alerts to the new my for enterprise systems. + local system_id system_secret system_type notifier_url notifier_user notifier_pass config_load ns-plug 2>/dev/null && { config_get system_id config system_id "" config_get system_secret config secret "" config_get system_type config type "" + config_get notifier_url config notifier_url "" } - - if [ "$system_type" = "enterprise" ] && [ -n "$system_id" ] && [ -n "$system_secret" ]; then - notifier_url="https://my.nethesis.it/proxy/alerts" - notifier_user="$system_id" - notifier_pass="$system_secret" - else - notifier_url="" - fi procd_open_instance procd_set_param command $PROG @@ -48,22 +40,25 @@ start_service() { procd_append_param command -remoteRead.url="$datasource_url" procd_append_param command -remoteWrite.url="$datasource_url" - # Always notify the local alert-proxy (handles unregistered machines gracefully) - procd_append_param command -notifier.url="http://127.0.0.1:9095" - # Also forward alerts to my.nethesis.it for registered enterprise systems if [ -n "$notifier_url" ]; then # Avoid leaking secret inside command line local pass_file="/var/run/vmalert/notifier.pass" mkdir -p /var/run/vmalert chmod 700 /var/run/vmalert - ( umask 077; printf '%s' "$notifier_pass" > "$pass_file" ) + ( umask 077; printf '%s' "$system_secret" > "$pass_file" ) procd_append_param command -notifier.url="$notifier_url" - procd_append_param command -notifier.basicAuth.username="$notifier_user" + procd_append_param command -notifier.basicAuth.username="$system_id" procd_append_param command -notifier.basicAuth.passwordFile="$pass_file" + else + procd_append_param command -notifier.blackhole fi - + + # config_load ns-plug above discards vmalert's parsed UCI state + config_load vmalert 2>/dev/null || true + config_list_foreach main additional_parameters append_params + procd_set_param stdout 1 procd_set_param stderr 1 procd_set_param respawn 3600 5 5