diff --git a/packages/banip/Makefile b/packages/banip/Makefile index 95c27b4ea..6b368bc43 100644 --- a/packages/banip/Makefile +++ b/packages/banip/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=banip PKG_VERSION:=1.8.10 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/packages/banip/files/README.md b/packages/banip/files/README.md index 8109069b0..bbf5df08a 100644 --- a/packages/banip/files/README.md +++ b/packages/banip/files/README.md @@ -255,6 +255,7 @@ The `report` sub-command accepts an output mode: `text` (default, human-readable | ban_resolver | option | - | external resolver used for DNS lookups, by default the local resolver/forwarder will be used | | ban_remotelog | option | 0 | enable the cgi interface to receive remote logging events | | ban_remotetoken | option | - | unique token to communicate with the cgi interface | +| ban_blockhook | option | - | full path of an external script, called once for every IP newly blocked by the log service | ## Examples @@ -545,6 +546,18 @@ Examples to transfer remote logging events from an internal server to banIP via Please note: for security reasons use this cgi interface only internally and only encrypted via https transfer protocol. +**External hook for newly blocked IPs** +banIP can notify an external program whenever the log service adds a new IP to a blocklist Set (disabled by default). Set `ban_blockhook` to the full path of an executable script, e.g. `/usr/libexec/my-block-hook`. The hook is called once per newly blocked IP, right after the nftables element has been added, with four positional arguments: + +``` + $1: the blocked IP address, e.g. '198.51.100.44' + $2: the protocol family, either 'v4' or 'v6' + $3: the log count that triggered the block (ban_logcount) + $4: the Set expiry time (ban_nftexpiry), '0s' if the block is permanent +``` + +Please note: the hook runs synchronously inside the single-threaded log service, so it has to be fast and non-blocking - queue the event and process it elsewhere, never do network I/O in the hook itself. Its exit code and output are ignored. + **Download options** By default banIP uses the following pre-configured download options: diff --git a/packages/banip/files/banip-functions.sh b/packages/banip/files/banip-functions.sh index 945ae9b35..e4fdb45d6 100755 --- a/packages/banip/files/banip-functions.sh +++ b/packages/banip/files/banip-functions.sh @@ -41,6 +41,7 @@ ban_mailprofile="ban_notify" ban_mailnotification="0" ban_remotelog="0" ban_remotetoken="" +ban_blockhook="" ban_nftloglevel="warn" ban_nftpriority="-100" ban_nftpolicy="memory" @@ -2695,7 +2696,7 @@ f_monitor() { case "${log_type}" in tail) "${ban_logreadcmd}" -qf "${ban_logreadfile}" 2>/dev/null | - "${ban_grepcmd}" -e "${ban_logterm}" 2>/dev/null + "${ban_grepcmd}" --line-buffered -e "${ban_logterm}" 2>/dev/null ;; logread) "${ban_logreadcmd}" -fe "${ban_logterm}" 2>/dev/null @@ -2835,6 +2836,12 @@ f_monitor() { fi block_cache="${block_cache} ${ip} " f_log "info" "add IP '${ip}' (cnt: ${ban_logcount}, expiry: ${ban_nftexpiry:-"0"}) to blocklist${proto} Set" + + # optional external hook, called once for every newly blocked IP + # + if [ -n "${ban_blockhook}" ] && [ -x "${ban_blockhook}" ]; then + "${ban_blockhook}" "${ip}" "${proto#.}" "${ban_logcount}" "${ban_nftexpiry:-0s}" >/dev/null 2>&1 + fi else f_log "info" "failed to add IP '${ip}' to blocklist${proto} Set with rc '${?}'" continue diff --git a/packages/ns-api/files/ns.threatshield b/packages/ns-api/files/ns.threatshield index 1fe714800..019c01c8a 100644 --- a/packages/ns-api/files/ns.threatshield +++ b/packages/ns-api/files/ns.threatshield @@ -462,7 +462,7 @@ def list_blocklist(e_uci): feed = feeds[f] enabled = f in enabled_feeds - if 'nethesis-blacklists' in feed.get('url_4'): + if 'nethesis' in feed.get('url_4'): type = 'enterprise' else: type = 'community' diff --git a/packages/ns-threat_shield/Makefile b/packages/ns-threat_shield/Makefile index c8413c7ea..ddf2c9817 100644 --- a/packages/ns-threat_shield/Makefile +++ b/packages/ns-threat_shield/Makefile @@ -7,7 +7,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=ns-threat_shield -PKG_VERSION:=1.0.1 +PKG_VERSION:=1.1.0 PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/ns-threat_shield-$(PKG_VERSION) @@ -46,6 +46,8 @@ define Package/ns-threat_shield/install $(INSTALL_DIR) $(1)/usr/libexec $(INSTALL_BIN) ./files/ts-dns $(1)/usr/sbin/ts-dns $(INSTALL_BIN) ./files/ts-ip $(1)/usr/sbin/ts-ip + $(INSTALL_BIN) ./files/ts-insights-report.py $(1)/usr/sbin/ts-insights-report + $(INSTALL_BIN) ./files/ts-insights-hook $(1)/usr/libexec/ts-insights-hook $(INSTALL_BIN) ./files/20_threat_shield $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/ts-dns.hook $(1)/usr/share/ns-plug/hooks/register/90ts-dns $(INSTALL_BIN) ./files/ts-dns.hook $(1)/usr/share/ns-plug/hooks/unregister/90ts-dns @@ -62,6 +64,7 @@ define Package/ns-threat_shield/install $(INSTALL_DIR) $(1)/etc/uci-defaults $(INSTALL_BIN) ./files/banip-defaults $(1)/etc/uci-defaults/99-nethsec-banip $(INSTALL_BIN) ./files/banip-extra-defaults $(1)/etc/uci-defaults/96-nethsec-banip-extra + $(INSTALL_BIN) ./files/banip-insights-defaults $(1)/etc/uci-defaults/97-nethsec-banip-insights $(INSTALL_BIN) ./files/35_ns-threat_shield $(1)/etc/uci-defaults/35_ns-threat_shield $(INSTALL_BIN) ./files/96_ns-threat_shield $(1)/etc/uci-defaults/96_ns-threat_shield gzip -9n $(1)/usr/share/threat_shield/nethesis-dns.sources @@ -84,6 +87,7 @@ define Package/ns-threat_shield/prerm if [ -z "$${IPKG_INSTROOT}" ]; then crontab -l | grep -v "/etc/init.d/banip reload" | sort | uniq | crontab - crontab -l | grep -v "/etc/init.d/adblock" | sort | uniq | crontab - + crontab -l | grep -v "/usr/sbin/ts-insights-report" | sort | uniq | crontab - fi exit 0 endef diff --git a/packages/ns-threat_shield/README.md b/packages/ns-threat_shield/README.md index 1773bbd5f..9422824f5 100644 --- a/packages/ns-threat_shield/README.md +++ b/packages/ns-threat_shield/README.md @@ -21,6 +21,7 @@ The following categories require a valid entitlement: - `yoroisusplvl1` (was `yoroi_souspicious_level1` on NS7) - `yoroisusplvl2` (was `yoroi_souspicious_level2` on NS7) - `nethesislvl3` (was `nethesis_level3` on NS7) +- `nethesisinsights` (attackers reported by the other Nethesis firewalls, see [Nethesis Insights](#nethesis-insights)) After machine registration, above categories will be automatically added to existing banip categories (`/etc/banip/banip.custom.feeds`). @@ -47,6 +48,42 @@ ts-ip /etc/init.d/banip restart ``` +### Nethesis Insights + +If the machine is registered, `ts-ip` also joins the [Nethesis Insights](https://github.com/nethesis/nethesis-insights) +threat shield: every IP blocked by the banip log service is reported to the Insights server, and the +list aggregated from the reports of all registered firewalls is blocked locally. +Both directions authenticate with the `system_id` and `secret` of the subscription: no additional +configuration is required and nothing is sent from a machine which is not registered. + +Reporting side: + +- `ts-ip` sets `banip.global.ban_blockhook` to `/usr/libexec/ts-insights-hook`; banip calls it once + for every IP added to a blocklist Set by the log service +- the hook only appends a JSON line to `/var/run/ns-insights/threat-events.jsonl`, so that a burst + of blocked IPs never slows down the banip log service +- `/usr/sbin/ts-insights-report` is executed every 5 minutes by cron: it sends the spooled events to + `POST /v1/threat-events` in batches of at most 500, then writes the outcome to + `/var/run/ns-insights/last_push.json` +- only globally routable addresses are reported: private, CGNAT, link-local, reserved and + documentation ranges are dropped locally, along with the events older than 2 hours +- on a failed push the events are kept in the spool and sent again at the next run, duplicated + reports are discarded by the server + +Blocking side: + +- the `nethesisinsights` feed points to `GET /v1/blocklist`, it is added to `ban_feed` on + registration and it is reloaded with all the other feeds every 4 hours +- an IP is published by the server only after it has been reported by several distinct firewalls, + and it expires when nobody reports it any more + +On unregistration the hook and the feed are both removed. + +Check the last report, example: +``` +cat /var/run/ns-insights/last_push.json +``` + ## ts-dns Threat shield DNS (`ts-dns`) is a special configuration for [adblock](https://github.com/openwrt/packages/tree/master/net/adblock). diff --git a/packages/ns-threat_shield/files/20_threat_shield b/packages/ns-threat_shield/files/20_threat_shield index 74349383e..806564e38 100644 --- a/packages/ns-threat_shield/files/20_threat_shield +++ b/packages/ns-threat_shield/files/20_threat_shield @@ -2,3 +2,4 @@ crontab -l | grep -q '/etc/init.d/banip' || echo '0 */4 * * * sleep $(( RANDOM % 3600 )); /etc/init.d/banip reload' >> /etc/crontabs/root crontab -l | grep -q '/etc/init.d/adblock' || echo '1 */12 * * * sleep $(( RANDOM % 3600 )); /etc/init.d/adblock reload' >> /etc/crontabs/root +crontab -l | grep -q '/usr/sbin/ts-insights-report' || echo '*/5 * * * * sleep $(( RANDOM % 60 )); /usr/sbin/ts-insights-report' >> /etc/crontabs/root diff --git a/packages/ns-threat_shield/files/banip-insights-defaults b/packages/ns-threat_shield/files/banip-insights-defaults new file mode 100755 index 000000000..6a6be5ee3 --- /dev/null +++ b/packages/ns-threat_shield/files/banip-insights-defaults @@ -0,0 +1,15 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# An already registered unit must join the Nethesis Insights threat shield right +# after an image update, without waiting for the next machine registration: +# ts-ip sets the banip block hook and adds the nethesisinsights feed. + +[ -n "$(uci -q get ns-plug.config.secret)" ] || exit 0 +[ -n "$(uci -q get banip.global.ban_blockhook)" ] && exit 0 + +/usr/sbin/ts-ip diff --git a/packages/ns-threat_shield/files/banip.nethesis.feeds b/packages/ns-threat_shield/files/banip.nethesis.feeds index 3ec2273a8..0fd5c63b2 100644 --- a/packages/ns-threat_shield/files/banip.nethesis.feeds +++ b/packages/ns-threat_shield/files/banip.nethesis.feeds @@ -23,6 +23,13 @@ "chain": "in", "descr": "Yoroi suspicious - Level 2" }, + "nethesisinsightslvl2": { + "url_4": "https://__USER__:__PASSWORD__@insights.nethesis.it/v1/blocklist", + "url_6": "https://__USER__:__PASSWORD__@insights.nethesis.it/v1/blocklist", + "rule": "feed 1", + "chain": "in", + "descr": "Nethesis Insights - Level 2" + }, "nethesislvl3": { "url_4": "https://__USER__:__PASSWORD__@bl.nethesis.it/plain/__TYPE__/nethesis-blacklists/nethesis_level3.netset", "rule": "feed 1", diff --git a/packages/ns-threat_shield/files/ts-insights-hook b/packages/ns-threat_shield/files/ts-insights-hook new file mode 100755 index 000000000..be39ad820 --- /dev/null +++ b/packages/ns-threat_shield/files/ts-insights-hook @@ -0,0 +1,26 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# banip ban_blockhook script: spool a threat event for nethesis-insights. +# +# Called by the banip log service for every newly blocked IP, arguments: +# $1 IP address, $2 protocol family (v4|v6), $3 log count, $4 Set expiry +# +# It must stay fast and non-blocking: the only job here is to append one JSON +# line to the spool file, /usr/sbin/ts-insights-report sends it later on. +# + +SPOOL_DIR="/var/run/ns-insights" +SPOOL="${SPOOL_DIR}/threat-events.jsonl" + +[ -n "$1" ] || exit 0 + +[ -d "${SPOOL_DIR}" ] || mkdir -p "${SPOOL_DIR}" + +printf '{"value":"%s","scope":"Ip","type":"ban","scenario":"nethsecurity/banip-log","origin":"banip","duration":"%s","created_at":"%s"}\n' \ + "$1" "${4:-0s}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" >>"${SPOOL}" diff --git a/packages/ns-threat_shield/files/ts-insights-report.py b/packages/ns-threat_shield/files/ts-insights-report.py new file mode 100755 index 000000000..9f5d2d8e7 --- /dev/null +++ b/packages/ns-threat_shield/files/ts-insights-report.py @@ -0,0 +1,222 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# +# Send the IPs blocked by the banip log service to the Nethesis Insights server. +# +# Events are spooled by /usr/libexec/ts-insights-hook (banip ban_blockhook) and +# pushed here in batches by cron. The unit is identified with the ns-plug +# subscription credentials; without them the script is a no-op. +# + +import base64 +import ipaddress +import json +import os +import shutil +import ssl +import syslog +import urllib.error +import urllib.request +from datetime import datetime, timedelta, timezone + +from euci import EUci + +SPOOL_DIR = "/var/run/ns-insights" +SPOOL = os.path.join(SPOOL_DIR, "threat-events.jsonl") +SENDING = SPOOL + ".sending" +STATUS = os.path.join(SPOOL_DIR, "last_push.json") + +DEFAULT_URL = "https://insights.nethesis.it" +MAX_DECISIONS_PER_REQUEST = 500 +MAX_SPOOLED_EVENTS = 5000 +MAX_EVENT_AGE = timedelta(hours=2) +TIMEOUT = 20 + + +def log(priority, message): + syslog.syslog(priority, message) + + +def read_config(uci): + """Return (base_url, system_id, secret, verify_tls) or None if the unit is not registered.""" + system_id = uci.get("ns-plug", "config", "system_id", default="") + secret = uci.get("ns-plug", "config", "secret", default="") + if not system_id or not secret: + return None + + base_url = uci.get("ns-plug", "config", "insights_url", default=DEFAULT_URL).rstrip( + "/" + ) + verify_tls = uci.get("ns-plug", "config", "tls_verify", default="1") != "0" + + return base_url, system_id, secret, verify_tls + + +def claim_spool(): + """Move the spool file aside so the hook can keep appending, then return its lines.""" + if os.path.exists(SPOOL): + if os.path.exists(SENDING): + # leftover from a previous failed run: keep the oldest events first + with open(SENDING, "a") as dst, open(SPOOL) as src: + shutil.copyfileobj(src, dst) + os.unlink(SPOOL) + else: + os.rename(SPOOL, SENDING) + + if not os.path.exists(SENDING): + return [] + + with open(SENDING) as spool: + return spool.readlines() + + +def is_reportable(value): + """Only public unicast addresses leave the firewall: never report local traffic.""" + try: + return ipaddress.ip_address(value).is_global + except ValueError: + return False + + +def parse_events(lines): + """Validate the spooled lines, returning (decisions, discarded_count).""" + decisions = [] + discarded = 0 + oldest = datetime.now(timezone.utc) - MAX_EVENT_AGE + + for line in lines: + line = line.strip() + if not line: + continue + try: + decision = json.loads(line) + value = decision["value"] + created_at = datetime.strptime( + decision["created_at"], "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc) + except (ValueError, KeyError, TypeError): + discarded += 1 + continue + + # the server promotes an IP only if it has been reported recently, stale events are useless + if not is_reportable(value) or created_at < oldest: + discarded += 1 + continue + + decisions.append(decision) + + if len(decisions) > MAX_SPOOLED_EVENTS: + discarded += len(decisions) - MAX_SPOOLED_EVENTS + decisions = decisions[-MAX_SPOOLED_EVENTS:] + + return decisions, discarded + + +def requeue(decisions): + """Put the decisions which have not been sent back into the spool.""" + if not decisions: + if os.path.exists(SENDING): + os.unlink(SENDING) + return + + with open(SENDING, "w") as spool: + for decision in decisions: + spool.write(json.dumps(decision) + "\n") + + +def send_batch(config, decisions): + """Post a batch of decisions, returning the server counters.""" + base_url, system_id, secret, verify_tls = config + payload = json.dumps( + {"schema_version": 1, "system_id": system_id, "decisions": decisions} + ).encode() + credentials = base64.b64encode(f"{system_id}:{secret}".encode()).decode() + request = urllib.request.Request( + f"{base_url}/v1/threat-events", + data=payload, + method="POST", + headers={ + "Content-Type": "application/json", + "Authorization": f"Basic {credentials}", + }, + ) + context = None if verify_tls else ssl._create_unverified_context() + + with urllib.request.urlopen(request, timeout=TIMEOUT, context=context) as response: + return json.loads(response.read()) + + +def write_status(status): + status["timestamp"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + with open(STATUS, "w") as status_file: + json.dump(status, status_file) + + +def main(): + syslog.openlog("ts-insights-report") + + uci = EUci() + config = read_config(uci) + if config is None: + return + + os.makedirs(SPOOL_DIR, exist_ok=True) + decisions, discarded = parse_events(claim_spool()) + if discarded: + log( + syslog.LOG_INFO, + f"discarded {discarded} invalid, stale or non-public events", + ) + if not decisions: + requeue([]) + return + + sent = stored = duplicates = 0 + for index in range(0, len(decisions), MAX_DECISIONS_PER_REQUEST): + batch = decisions[index : index + MAX_DECISIONS_PER_REQUEST] + try: + counters = send_batch(config, batch) + except (urllib.error.URLError, OSError, ValueError) as error: + # keep the events for the next run, the server deduplicates redeliveries + requeue(decisions[index:]) + log( + syslog.LOG_WARNING, + f"push of {len(decisions) - index} events failed: {error}", + ) + write_status( + { + "success": False, + "sent": sent, + "stored": stored, + "duplicates": duplicates, + "error": str(error), + } + ) + return + + sent += len(batch) + stored += counters.get("stored", 0) + duplicates += counters.get("duplicates", 0) + + requeue([]) + log( + syslog.LOG_INFO, + f"pushed {sent} events (stored: {stored}, duplicates: {duplicates})", + ) + write_status( + { + "success": True, + "sent": sent, + "stored": stored, + "duplicates": duplicates, + "error": None, + } + ) + + +if __name__ == "__main__": + main() diff --git a/packages/ns-threat_shield/files/ts-ip b/packages/ns-threat_shield/files/ts-ip index 8acc85333..ad83f106d 100755 --- a/packages/ns-threat_shield/files/ts-ip +++ b/packages/ns-threat_shield/files/ts-ip @@ -37,6 +37,13 @@ if [ ! -z "$SYSTEM_SECRET" ] && [ ! -z "$SYSTEM_ID" ]; then uci add_list banip.global.ban_allowurl="https://$SYSTEM_ID:$SYSTEM_SECRET@bl.nethesis.it/plain/$TYPE/nethesis-blacklists/whitelist.global" uci commit banip fi + # report the IPs blocked by the banip log service and consume the aggregated + # blocklist built by Nethesis Insights from the reports of the whole fleet + uci set banip.global.ban_blockhook="/usr/libexec/ts-insights-hook" + if ! uci -q get banip.global.ban_feed | grep -q -w nethesisinsights; then + uci add_list banip.global.ban_feed="nethesisinsights" + fi + uci commit banip else allow=$(uci -q get banip.global.ban_allowurl | tr " " "\n" | grep bl.nethesis.it) if [ "$allow" != "" ]; then @@ -52,4 +59,10 @@ else uci commit banip fi > /etc/banip/banip.custom.feeds + # stop reporting to Nethesis Insights, the nethesisinsights feed is already + # removed along with the other enterprise feeds + if [ -n "$(uci -q get banip.global.ban_blockhook)" ]; then + uci -q delete banip.global.ban_blockhook + uci commit banip + fi fi