diff --git a/.agents/skills/netifyd/SKILL.md b/.agents/skills/netifyd/SKILL.md new file mode 100644 index 000000000..48f2fe6c7 --- /dev/null +++ b/.agents/skills/netifyd/SKILL.md @@ -0,0 +1,259 @@ +--- +name: netifyd +description: Work with the Netify Agent (netifyd) DPI daemon on NethSecurity — its configuration graph, processor/sink plugins, flow-actions rules, criteria expressions, address groups, telemetry records and CLI. Always use this skill when the task touches netifyd, netify-proc-*/netify-sink-* plugins, /etc/netifyd, netifyd.conf, plugins.d, address-groups.d, DPI criteria or flow expressions, nfqueue capture, conntrack labels set by DPI, or netify telemetry (flow, flow-purge, flow-stats, agent-status, intelligence) — even when the user only says "DPI", "flow actions", "netify", "app blocking" or "the agent". It fetches the live Netify v5 documentation index instead of relying on memory, and maps out which file on a NethSecurity device owns what. +compatibility: Works with OpenCode and other Agent Skills-compatible tools. Needs network access for the Netify docs index, and SSH to a live device for anything that must be verified against a running agent. +metadata: + domain: nethsecurity-dpi + type: daemon-integration +--- + +## What I do + +- Route you to the authoritative Netify Agent v5 documentation before you write anything +- Explain netifyd's configuration graph as it actually exists on NethSecurity: who owns which file, and what a reload re-reads +- Get processor/sink plugin wiring right, including the two different ways a sink is referenced +- Point criteria expressions at the field and type authority instead of at guesswork +- Give you the on-device verification loop: the dumps, reload vs restart, and where to watch + +## When to use me + +Any task involving: netifyd configuration or plugins, flow-actions rules, criteria/expression authoring, address groups, category lists, application overlay, capture interfaces and nfqueue, netify telemetry records, or debugging a netifyd that crash-loops or classifies nothing. + +Not for: bumping the `netifyd` package version or touching `netify-dist.mk` — that's `openwrt-package`. Writing the `ns.dpi` API surface — that's `ns-api`. + +--- + +## Step 1 — Fetch the live docs index first + +Netify's agent docs move between releases, and the v5 plugin/telemetry surface is large enough that +recalling field names from memory is how configs get written with keys that silently do nothing. So the +first action of every netifyd task, before opening any file: + +``` +WebFetch https://www.netify.ai/developer/agent/v5/llms.txt +``` + +That is a link map, not documentation — it lists every page and schema with a one-line summary. Read it, +pick the one to three pages that actually cover the task, and fetch those. Do not fetch the whole set. + +Docs are machine-readable Markdown at `…/developer/agent/v5/docs/.md`; JSON Schemas for plugin +configs and telemetry records are at `…/developer/agent/v5/schemas/.json`. Prefer these over the +human-facing `netify.ai/documentation/…` HTML pages — same content, less noise. Older notes in this repo +still link the HTML paths; treat them as equivalent, and use the index to find the current URL. + +Which page to fetch, by task: + +| Task | Fetch | +|---|---| +| Writing or reviewing a criteria expression | Expression Engine (**always** — it is the field authority) | +| Blocking / QoS / enforcement rules | Flow Actions Processor + Expression Engine | +| Adding a telemetry consumer | Core or Aggregator Processor + the relevant Sink + the telemetry record page | +| Parsing telemetry someone else emitted | the matching Telemetry page, plus its schema if you need every field | +| Capture setup, nfqueue, interface roles | Network Interfaces | +| Runtime tuning (flow table, caches, threads) | Agent Settings | +| Reusable IP/MAC groups | Address Groups | +| Custom domain/regex/CIDR categories | Category Lists (BYOC) | +| Relabelling detected apps | Application Overlay | +| Verifying a config key exists at all | the plugin's JSON Schema under `schemas/` | + +`…/developer/agent/v5/llms-full.txt` is the comprehensive dump. Reach for it only when the task spans many +plugins at once, or when a specific page turns out not to answer the question — it is large. + +**Check the version.** The index is v5; the agent this repo ships is pinned in +`packages/netifyd/Makefile` (`NETIFYD_VERSION`), and a live device reports its own via +`netifyd --version`. If they disagree at the minor level, say so before relying on a newly documented +option — the docs describe the current v5, not necessarily the build in the tree. + +--- + +## Step 2 — Know the configuration graph + +netifyd reads one entry-point file and follows it into directories. Nothing merges implicitly; every +piece is loaded because something named it. + +``` +/etc/netifyd.conf entry point: names the profile, the state paths, the PLM library +└── /etc/netifyd/profiles.d/00-default.conf + the actual [netifyd] tuning: flow map, TTLs, caches, protocols, + max_detection_pkts, netify-api, netlink buffers +/etc/netifyd/ +├── interfaces.d/10-nfqueue.conf capture sources — on NethSecurity these are nfqueue, not devices +├── plugins.d/*.conf plugin LOADERS (ini). One section = one plugin instance +├── netify-.json per-instance plugin config, named by its loader's conf_filename +├── address-groups.d/NN-.conf @tag groups; one address per line +├── categories.d/NN-.conf BYOC pattern lists (dom:/rxp:/net: entries) +├── netify-apps.conf application signatures +├── netify-categories.json category definitions +├── netify-*-catalog.json display metadata, refreshed nightly by dpi-data-update +└── agent.uuid agent identity (conffile — do not regenerate casually) +``` + +In this repo those files live under `packages/netifyd/files/etc/…`, plus +`packages/ns-monitoring/files/netifyd/…` for the monitoring consumers and `packages/ns-dpi/files/…` for +the DPI pipeline. `/usr/share/netifyd/` holds the shipped templates and `functions.sh`; treat it as +read-only reference at runtime. + +### Never hand-edit a generated file + +Several files under `/etc/netifyd/` are written by NethSecurity code on every run, so an edit survives +until the next reload and then vanishes. Change the generator or its UCI input instead. + +| File | Written by | Real input | +|---|---|---| +| `netify-proc-flow-actions.json` | `/usr/sbin/dpi-config` (via `/usr/sbin/dpi`) | `/etc/config/dpi` | +| `table inet netifyd` (nftables) | `/usr/sbin/ns-netifyd-configure` (on start and reload) | `/etc/config/netifyd` → `ns_config` | +| `netify-application-catalog.json` and the other catalog/category files | `dpi-data-update` (nightly cron) | Netify's data service | +| `/usr/share/nftables.d/table-pre/` DPI chains | `/usr/sbin/dpi-nft` | `/etc/config/dpi` | + +`/usr/sbin/dpi` is the whole DPI apply path: `dpi-config` → `dpi-nft` → `/etc/init.d/netifyd reload`. +Run it after changing `/etc/config/dpi`; don't reproduce its steps by hand. + +--- + +## Plugin loaders — the part the docs assume you already know + +A plugin is not enabled by writing its JSON. It is enabled by an ini section in `plugins.d/`: + +```ini +[proc-ns-flows] +enable = yes +plugin_library = ${path_plugin_libdir}/libnetify-proc-core.so.0.0.0 +conf_filename = ${path_state_persistent}/netify-ns-flows-proc.json +``` + +Three things follow from this, and each one is a real mistake if missed: + +**The section name is the instance name.** It is the handle every other config uses to point at this +plugin. Rename the section and every reference breaks silently. + +**One library can be loaded many times under different names.** `ns-monitoring` loads +`libnetify-proc-core.so` as `proc-ns-flows` and `libnetify-proc-aggregator.so` as `proc-ns-stats`, each +with its own HTTP sink instance (`sink-ns-flows`, `sink-ns-stats`) and its own JSON. That is the pattern to +copy for a new consumer: your own loader file, your own instance names, your own JSON — never bolt a +channel onto someone else's instance, because then their reload semantics and failure modes become yours. + +**A sink is referenced in two different shapes**, depending on who is doing the referencing: + +- From a **processor** config — a nested map, `sinks` → sink instance → channel: + ```json + { "sinks": { "sink-ns-flows": { "default": { "enable": true, "types": ["stream-flows"] } } } } + ``` +- From a **flow-actions target** — flat `sink` and `channel` keys: + ```json + { "log": { "target_type": "sink", "target_enabled": true, "sink": "sink-log", "channel": "nfa_block_log" } } + ``` + +The two are different schemas, so check which one you are writing against — the processor schema for a +proc config, the flow-actions schema for a target — and confirm delivery on a device rather than assuming +a loaded plugin is a delivering one. + +Channel names inside a sink config are yours to choose, but the processor or target that routes to them +must use the exact same string. + +--- + +## Criteria expressions + +Fetch the Expression Engine page before writing one. It carries the field list and the type table, and +those are the two things that decide whether an expression is valid — neither is guessable, and both are +what a generator has to encode. + +**Terminate every expression with `;`.** Standard syntax requires the terminator; only the compact syntax +does without. Normalise it on everything you emit, whatever the source — the current generator appends it +to criteria it builds itself, but not to raw `criteria` taken from UCI and not to entries in the global +`exemptions` array, so both of those paths need it added. + +**Quote according to the type table, not by analogy.** Address-typed fields take bare literals; only +`string` and `mixed` types are quoted: + +``` +local_ip == 10.0.2.0/24 # address type: no quotes +app == 'netify.youtube' # string type: quoted +local_ip == @objects_ns_hostset_1 # @tag reference: no quotes +``` + +**Only emit fields the Expression Engine documents,** and whitelist them in a generator rather than +passing user input straight through. Values are looser than fields — an app, protocol or category tag the +agent doesn't know simply never matches. + +**Match by name, never by numeric ID.** Use `app`, `proto`, `app_category`, `proto_category`, not the +`_id` variants. Signature and category data is refreshed nightly by `dpi-data-update` and is free to +renumber IDs; names stay readable in stored config and survive the refresh. + +**Never emit an empty criteria.** Validate before writing and skip-and-log what fails, rather than writing +it out and finding out later. + +--- + +## Address groups + +A group is a file whose **name defines the tag**: netifyd strips the two-digit prefix and the `.conf` +suffix, so `/etc/netifyd/address-groups.d/30-objects_ns_hostset_1.conf` is referenced as +`local_ip == @objects_ns_hostset_1`. Contents are one address, CIDR or MAC per line. + +Why this matters: a criteria stays one term wide however large the group gets, and editing group membership +touches one small file instead of regenerating the whole plugin config. A reload re-reads every group file. + +Two operational facts: + +- **The directory does not exist by default.** netifyd logs `Error opening directory` on every start until + something creates it. Create it in the package install and re-ensure it in any writer. +- **Prefixes below `20-` belong to the agent**, which manages its own groups over its control socket. + NethSecurity writers start at `20-`. Each writer must delete only files in its own range, or it will + destroy agent-managed and hand-placed groups. + +MAC members only match via `mac ==`, so don't put them in a group a `local_ip` rule references. + +--- + +## Verify on a live device + +The agent's own dumps are the authority on what it can currently match: + +```bash +netifyd --version +netifyd --status # PID, uptime, flow counts +netifyd --dump-apps # applications the running agent can match +netifyd --dump-protos +netifyd --dump-categories # category TAGS — e.g. social-media, not social +netifyd --dump-category +``` + +Use the dumps, not the catalog JSONs, to decide whether a tag can match. The catalogs carry display names +and icons for ~1500+ apps; the *loaded signature list* drops to ~200 without a subscription (`ns-dpi`'s +`70dpi` strips premium signatures on unregister). A catalog entry with no loaded signature never matches. + +Applying and watching: + +```bash +/etc/init.d/netifyd reload # re-runs ns-netifyd-configure, then signals the agent: re-reads + # plugin configs, address groups and category lists. Enough for + # almost every config change. +/etc/init.d/netifyd restart # only for netifyd.conf, the profile, or interfaces.d changes +/usr/sbin/dpi # after editing /etc/config/dpi — regenerate, apply nft, reload +logread -f -e netifyd # watch a reload land, and whether the agent stays up +nft list table inet netifyd # the nfqueue capture table (bypass sets, queue chains) +``` + +Then generate traffic that should match and confirm the outcome — for enforcement, that the conntrack entry +carries the expected label from `/etc/connlabel.conf`; for telemetry, that records reach the consumer. + +--- + +## Before you ship a config change + +- Fetched the relevant docs page this session, rather than working from recall +- Every key checked against the plugin's JSON Schema — a wrong key is silent +- Every expression semicolon-terminated, quoting per the type table, fields whitelisted, matched by name +- Editing the generator or its UCI input, not a generated file +- Reload vs restart chosen deliberately, and the agent still running afterwards (`logread` clean) +- Traffic actually generated, and the label or the telemetry record confirmed +- New consumer got its own loader section, instance names and JSON, and did not attach to an existing instance + +--- + +## References + +- `references/nethsecurity-wiring.md` — the full NethSecurity-side inventory: UCI options, the nfqueue + table and bypass sets, the DPI pipeline and conntrack labels, telemetry consumers, HA triggers. Read it + when the task crosses from the daemon into NethSecurity's own plumbing. diff --git a/.agents/skills/netifyd/references/nethsecurity-wiring.md b/.agents/skills/netifyd/references/nethsecurity-wiring.md new file mode 100644 index 000000000..1876e961f --- /dev/null +++ b/.agents/skills/netifyd/references/nethsecurity-wiring.md @@ -0,0 +1,202 @@ +# netifyd on NethSecurity — the surrounding plumbing + +What NethSecurity puts around the daemon. Read this when a task crosses from netifyd's own configuration +into the repo's packages, UCI, nftables or HA. Paths are repo paths; the on-device path is the part after +`files/`. + +## Table of contents + +- [Package layout](#package-layout) +- [Capture: nfqueue, not interfaces](#capture-nfqueue-not-interfaces) +- [UCI: /etc/config/netifyd](#uci-etcconfignetifyd) +- [The DPI pipeline](#the-dpi-pipeline) +- [Conntrack labels](#conntrack-labels) +- [Telemetry consumers](#telemetry-consumers) +- [Data refresh and licensing](#data-refresh-and-licensing) +- [Service lifecycle and HA](#service-lifecycle-and-ha) +- [Conffiles](#conffiles) + +## Package layout + +`packages/netifyd/` — the agent itself. Prebuilt binaries and plugin libraries are downloaded per +architecture (`netify-dist.mk` carries the filenames and hashes; regenerated by +`tools/netifyd-update/netifyd-update.py`), so there is no compile step. Ships: + +- `files/etc/netifyd.conf`, `files/etc/netifyd/profiles.d/00-default.conf` — agent entry point and tuning +- `files/etc/netifyd/interfaces.d/10-nfqueue.conf` — capture sources +- `files/etc/netifyd/plugins.d/10-netify-*.conf` + matching `netify-*.json` — the shipped plugin + instances: `proc-core` (disabled by default), `proc-aggregator`, `proc-flow-actions`, `sink-log`, + `sink-http` +- `files/usr/sbin/ns-netifyd-configure.py` → `/usr/sbin/ns-netifyd-configure` — the nftables capture table +- `files/etc/uci-defaults/99-netify-*` — first-boot migrations and defaults (nfqueue enable, autoconfig + off, coredumps off, v4→v5 migration) + +`packages/ns-dpi/` — DPI filtering: the flow-actions generator, the nft enforcement chains, the conntrack +label map, and the nightly data/license updaters. + +`packages/ns-monitoring/` — telemetry consumers (`ns-flows`, `ns-stats`), each with its own plugin loader +pair and JSON. + +Version bumps and `netify-dist.mk` regeneration are `openwrt-package` territory, not this skill's. + +## Capture: nfqueue, not interfaces + +The agent does not capture on devices here. `interfaces.d/10-nfqueue.conf` declares two nfqueue capture +sources with roles: + +| Section | role | queue_id | instances | +|---|---|---|---| +| `capture-interface-lan` | lan | 50 | 4 (50–53) | +| `capture-interface-wan` | wan | 54 | 4 (54–57) | + +`conntrack_counters = true` on both. Packets arrive because `/usr/sbin/ns-netifyd-configure` builds +`table inet netifyd` and queues them there. That script is the **only** writer of that table and +re-renders it wholesale on every run (start and reload), from UCI. + +Per chain it short-circuits, in order: `lo`, `ct state untracked`, the `nfq_bypass_v4`/`nfq_bypass_v6` +sets, then `ct packets > 32` — because the agent only inspects the first 32 packets of a flow +(`max_detection_pkts` in the profile), so queueing the rest is pure overhead. `nfq_input` additionally +skips `ct direction reply` when the `output` chain is absent, so the firewall's own DNS responses and ping +replies don't get attributed as remote traffic. + +Consequence for capture-side changes: a "bypass" at the nfqueue layer means *not analysed at all* — no +telemetry, no flow record, invisible in ns-flows. That is a different thing from exempting traffic from +enforcement, which belongs in flow-actions `exemptions`. + +## UCI: /etc/config/netifyd + +Section `ns_config 'config'` (NethSecurity's own; the upstream `netifyd` sections drive the init script): + +- `bypassv4` / `bypassv6` — list. Each entry is an address or CIDR, optionally `addr | description`; + the description becomes an nftables per-element `comment`. +- `firewall_traffic` — list of `input`, `output`, `forward`: which capture chains exist. Missing or + all-invalid falls back to all three, with a warning. Invalid values are skipped with a warning. + +Apply with `uci commit netifyd && reload_config` (the init script's `reload_service` re-runs +`ns-netifyd-configure`, then signals the agent). + +Upstream `netifyd` section options the init script consumes: `enabled`, `autoconfig` (forced off here by +uci-default, since capture is nfqueue), `options` list (extra CLI args), `internal_if` / `external_if` +lists. `procd_set_param file /etc/netifyd.conf` means procd restarts the service when that file changes. + +## The DPI pipeline + +`/usr/sbin/dpi` is the apply path, in order: + +1. `/usr/sbin/dpi-config` — reads `/etc/config/dpi`, writes `/etc/netifyd/netify-proc-flow-actions.json` +2. `/usr/sbin/dpi-nft` — writes the `dpi_actions` / `dpi_dummy` chains into + `/usr/share/nftables.d/table-pre/` +3. `/etc/init.d/netifyd reload` + +Current generator behaviour worth knowing before touching it: + +- `valid_actions` is `block`, `bulk`, `best_effort`, `video`, `voice`; anything else, or a disabled rule, + is skipped. +- A rule's raw `criteria` takes precedence over generated source/application/protocol/category matching. + Double quotes in it are rewritten to single quotes. **No `;` is appended to raw criteria** — generated + criteria get one, raw ones don't. Same for entries in the global `exemptions` array. +- A `device` naming a VLAN device causes `vlan_id == N && ` to be prepended — including to raw criteria, + so a rule carrying both its own `criteria` and a `device` gets the term whether or not it already has one. +- A hidden `analyzed` action (`detection_guessed || detection_complete;`) is appended last, labelling every + analysed flow. +- `dpi.config.firewall_exemption` optionally pre-fills `exemptions` with every firewall interface IP; + `exemption` sections add more, and an exemption whose criteria is an object ID is expanded to its IPs. +- `dpi.config.log_blocked` drives the nft log rule in `dpi-nft`, not any netify logging target. + +Enforcement is nft reading conntrack labels, not netifyd acting directly: netifyd classifies and sets a +label via its `ctlabel` target, and `dpi_actions` (hook prerouting, `filter + 10`) rejects labelled +traffic (`netify-blocked`) or DSCP-marks it (the QoS labels — the DPI→qosify bridge). + +## Conntrack labels + +`/etc/connlabel.conf`, shipped by `ns-dpi`. Bits are a compatibility contract: conntrack entries alive +across an upgrade keep their labels, so **never reassign an existing bit** — take the next free one. + +| Bit | Label | +|---|---| +| 0 | `netify-init` | +| 1 | `netify-blocked` | +| 2 | `netify-analyzed` | +| 3 | `bulk` | +| 4 | `best_effort` | +| 5 | `video` | +| 6 | `voice` | + +`netify-analyzed` is consumed by no nft rule and read only by `ns.flows` for display — it is a diagnostic +that tells support whether netifyd actually analysed a flow. "Nothing reads it in code" is not a reason to +drop it. + +## Telemetry consumers + +`packages/ns-monitoring/files/netifyd/` is the reference pattern for adding a consumer: one loader file +declaring a processor instance and a sink instance, plus one JSON each. + +| Consumer | Processor instance / library | Sink instance / library | Destination | +|---|---|---|---| +| ns-flows | `proc-ns-flows` / `libnetify-proc-core.so` | `sink-ns-flows` / `libnetify-sink-http.so` | `http://127.0.0.1:8080/flows`, types `stream-flows` + `stream-stats` | +| ns-stats | `proc-ns-stats` / `libnetify-proc-aggregator.so` | `sink-ns-stats` / `libnetify-sink-http.so` | `http://127.0.0.1:8081/stats` | + +Both sinks are loopback HTTP with `tls_verify: false` — fine because they never leave the host; do not +copy that setting to anything that does. The aggregator's `aggregator: 3`, `batched_rows: 100` and +`log_interval: 10` set the rollup granularity and batch size. + +The agent's shipped `sink-log` instance writes aggregator stats to `/var/run/netifyd` — tmpfs, and +`overwrite: true` on that channel. Useful for debugging; not a delivery mechanism to build on. + +`proc-core` ships **disabled** in `plugins.d/10-netify-proc-core.conf` (`enable = no`) with its sink +entries `enable: false`. `ns-monitoring` loads its own core instance instead. Enabling the shipped one to +get flows is the wrong lever — it doubles the processor, not the routing. + +## Data refresh and licensing + +`ns-dpi` runs two cron jobs (`99-dpi-data-update-cron`, `99-dpi-license-update-cron` uci-defaults): + +- `dpi-data-update` refreshes `netify-application-categories.json`, `netify-application-catalog.json`, + `netify-protocol-categories.json`, `netify-protocol-catalog.json` into `/etc/netifyd`. These are display + metadata — names, icons, categorisation — and they change nightly, which is why criteria must match by + name rather than by numeric ID. +- `dpi-license-update` handles subscription state. `70dpi` strips premium signatures on unregister, + dropping the matchable application set from ~1500+ to ~200 while the catalogs still list everything. So + catalog presence ≠ matchability; `netifyd --dump-apps` is the authority. + +## Service lifecycle and HA + +`START=99`, `STOP=1`, procd-managed, `respawn 3600 5 0`, `term_timeout 35`. The long term timeout is +because the agent lingers until flows expire unless `auto_flow_expiry` is set (it is, in the profile). + +`start_service` creates `/var/run/netifyd`, runs `ns-netifyd-configure`, then loads UCI. +`reload_service` re-runs `ns-netifyd-configure` and sends the reload signal — so a reload rebuilds the +nftables capture table *and* makes the agent re-read plugin configs, address groups and category lists. +`load_modules` (in `functions.sh`) modprobes `nfnetlink` and `nf_conntrack_netlink`. + +Every telemetry consumer is a plugin inside this one process, so the agent's availability is also +ns-flows', the stats aggregator's and the dashboards' — worth remembering when judging the blast radius of +a netifyd config change. + +HA: `ns-ha` carries `/etc/hotplug.d/keepalived/800-netifyd`, so netifyd state follows VRRP transitions. +Anything that regenerates netifyd config on address changes must order itself *after* 800 to see the final +service state, and must leave a stopped agent's files correct for the next promotion — write the file, +reload only if it changed and the agent is running. + +## Conffiles + +`packages/netifyd/Makefile` marks these as conffiles, i.e. preserved across sysupgrade: + +``` +/etc/config/netifyd +/etc/netifyd.conf +/etc/netifyd/agent.uuid +/etc/netifyd/netify-proc-core-auto.json +/etc/netifyd/netify-sink-http-auto.json +/etc/netifyd/plugins.d/99-netify-proc-core-auto.conf +/etc/netifyd/plugins.d/99-netify-sink-http-auto.conf +``` + +The `*-auto*` files are the Netify Informatics/cloud integration path, seeded from +`/usr/share/netifyd/`. `auto_informatics = no` in `netifyd.conf` and the +`99-netify-disable-autoconfig` uci-default keep them dormant. + +Files **not** in that list are replaced on upgrade — including everything under `plugins.d/` that isn't +`99-*-auto`, the per-plugin JSONs, `interfaces.d/` and the profile. A device-local edit to any of those is +lost on upgrade, which is another reason changes belong in the package. `/etc/config/dpi` does survive +(`sysupgrade -l` lists it). diff --git a/.gitignore b/.gitignore index 40665e2e7..58099fb39 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ build-logs build.conf netify-flow-actions netify-agent-stats-plugin -scripts/netifyd-apks +tools/netifyd-update/work/ +tools/netifyd-update/dist/ diff --git a/AGENTS.md b/AGENTS.md index a0e4eb0e7..53003e398 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project Overview -**NethSecurity** is an OpenWrt-based Linux firewall, built inside a rootless Podman container. Versions: OpenWrt `v24.10.5`, NethSecurity `8.7.2` (from `build.conf.example`). +**NethSecurity** is an OpenWrt-based Linux firewall, built inside a rootless Podman container. Read `build.conf.example` for the current versions — `OWRT_VERSION` (OpenWrt) and `NETHSECURITY_VERSION` (NethSecurity). Never assume either from memory or from versions quoted elsewhere; check upstream sources against the tag `OWRT_VERSION` names. --- @@ -10,6 +10,7 @@ This project has domain-specific skills available. You MUST activate the relevant skill whenever you work in that domain—don't wait until you're stuck. +- `netifyd` — **ACTIVATE** when working with the netifyd DPI daemon. Triggers: `/etc/netifyd` config, `plugins.d` loaders, `netify-proc-*`/`netify-sink-*` plugins, flow-actions rules, criteria/flow expressions, address groups, nfqueue capture, DPI conntrack labels, netify telemetry, or when user mentions netifyd, netify, DPI or flow actions. Fetches the live Netify v5 docs index; covers the config graph, plugin wiring, and which files are generated. - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. - `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt packages from the upstream feed (adblock, mwan3, banip, etc.). Triggers: updating non-ns- packages, comparing local forks against openwrt/packages, merging upstream improvements, or when user mentions upstream package updates. Auto-discovers packages; extracts old/new snapshots for side-by-side comparison; guides cross-package impact detection. @@ -71,6 +72,8 @@ recent host Ruby (4.x), so the site is built in a `ruby:3.3` container and the | `files/` | Filesystem overlay copied verbatim into image | | `patches/` | Patches applied to upstream OpenWrt feeds | | `.github/workflows/` | CI/CD (build-image, cleanup, docs, etc.) | +| `.agents/skills/` | Agent skills (`/SKILL.md`) | +| `tools/` | Maintenance helpers, e.g. `netifyd-update/netifyd-update.py` | --- @@ -82,6 +85,7 @@ recent host Ruby (4.x), so the site is built in a `ruby:3.3` container and the - **Do not set `PKG_SOURCE_URL`** when package code lives in this repo. Set it only when fetching from external GitHub releases. - To add a package to the image, create a corresponding `config/.conf` fragment that enables it at build time. - Renovate manages external package versions via magic comments in Makefiles: `# renovate: datasource=github-tags depName=Org/Repo` +- `packages/netifyd` is download-only: prebuilt per-arch binaries, filenames and hashes in `netify-dist.mk`, regenerated by `tools/netifyd-update/netifyd-update.py`. No compile step. --- @@ -204,6 +208,7 @@ Existing Python files (listed in `ruff.toml` under `extend-exclude`) are grandfa | [openwrt/packages](https://github.com/openwrt/packages) | OpenWrt package repository | | [NethServer/nethsecurity-controller](https://github.com/NethServer/nethsecurity-controller) | Firewall registration, VPN/proxy routing, and how the firewall connects to and calls the controller. | | [NethServer/ns8-nethsecurity-controller](https://github.com/NethServer/ns8-nethsecurity-controller) | NS8 deployment details, extra Loki/Grafana/Prometheus/WebSSH components, and controller packaging. | +| [Netify Agent v5 docs index](https://www.netify.ai/developer/agent/v5/llms.txt) | Machine-readable link map for netifyd: plugins, telemetry, expression engine, JSON schemas. Prefer over `netify.ai/documentation/…` HTML | **NethServer shared handbook** (follow for all contributions): diff --git a/packages/netifyd/Makefile b/packages/netifyd/Makefile index 4610eab05..e816a8ba2 100644 --- a/packages/netifyd/Makefile +++ b/packages/netifyd/Makefile @@ -1,16 +1,19 @@ include $(TOPDIR)/rules.mk PKG_NAME:=netifyd -NETIFYD_VERSION:=5.2.6 +NETIFYD_VERSION:=5.2.9 PKG_VERSION:=$(NETIFYD_VERSION) -PKG_RELEASE:=5 +PKG_RELEASE:=1 PKG_MAINTAINER:=Darryl Sokoloski PKG_LICENSE:=Unlicensed # Base URL for downloads -NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(PKG_VERSION)/25.12.2/$(ARCH) +NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(NETIFYD_VERSION)/25.12.5/$(ARCH) DL_DIR:=$(DL_DIR)/netifyd-$(PKG_VERSION)-$(ARCH) +# per-arch download filenames and hashes, regenerated by tools/netifyd-update/netifyd-update.py +include $(CURDIR)/netify-dist.mk + include $(INCLUDE_DIR)/package.mk define Package/netifyd @@ -19,6 +22,7 @@ define Package/netifyd TITLE:=Netify Binary Integration Package URL:=https://netify.ai/ DEPENDS:= \ + @(aarch64||x86_64) \ +ca-bundle \ +kmod-ipt-conntrack-label \ +kmod-ipt-ipset \ @@ -37,7 +41,6 @@ define Package/netifyd +libopenssl \ +libpcap \ +libpthread \ - +libsqlite3 \ +libstdcpp \ +libubox \ +libubus \ @@ -64,105 +67,78 @@ endef define Download/libnetifyd URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetifyd.so.4.0.0 - FILE:=libnetifyd.so.4.0.0 - HASH:=ffbbe5078a2b6575db4c478cf1b4d257f9b7241797c84aa1bcbca17ee1b23f3b + URL_FILE:=$(NETIFYD_FILE_LIBNETIFYD) + FILE:=$(NETIFYD_FILE_LIBNETIFYD) + HASH:=$(NETIFYD_HASH_LIBNETIFYD) endef $(eval $(call Download,libnetifyd)) define Download/libnetify-plm URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-plm.so.1.0.0 - FILE:=libnetify-plm.so.1.0.0 - HASH:=4320235873539f561238c92094702e74b75b8939a301b694255a5512fa036a00 + URL_FILE:=$(NETIFYD_FILE_PLM) + FILE:=$(NETIFYD_FILE_PLM) + HASH:=$(NETIFYD_HASH_PLM) endef $(eval $(call Download,libnetify-plm)) define Download/libnetify-proc-aggregator URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-proc-aggregator.so.0.0.0 - FILE:=libnetify-proc-aggregator.so.0.0.0 - HASH:=736acb4c22d891da66694eee5507ea70679885f0747dec7a07eec8c926dd76fc + URL_FILE:=$(NETIFYD_FILE_PROC_AGGREGATOR) + FILE:=$(NETIFYD_FILE_PROC_AGGREGATOR) + HASH:=$(NETIFYD_HASH_PROC_AGGREGATOR) endef $(eval $(call Download,libnetify-proc-aggregator)) define Download/libnetify-proc-core URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-proc-core.so.0.0.0 - FILE:=libnetify-proc-core.so.0.0.0 - HASH:=8c9a1f26a498a6d1c88d76e6aa0367749779ca5f44ea3fa614bc1814387258ed + URL_FILE:=$(NETIFYD_FILE_PROC_CORE) + FILE:=$(NETIFYD_FILE_PROC_CORE) + HASH:=$(NETIFYD_HASH_PROC_CORE) endef $(eval $(call Download,libnetify-proc-core)) -define Download/libnetify-proc-dev-discovery - URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-proc-dev-discovery.so.0.0.0 - FILE:=libnetify-proc-dev-discovery.so.0.0.0 - HASH:=756c01e22c3724311eb6952ca6d580c728592771948207b93bab9b9795c38d1c -endef -$(eval $(call Download,libnetify-proc-dev-discovery)) - define Download/libnetify-proc-flow-actions URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-proc-flow-actions.so.0.0.0 - FILE:=libnetify-proc-flow-actions.so.0.0.0 - HASH:=1d6f03ead8760e314f98f5fe083515b19bf9c60e720d36bc03f70fa60a9b0d2b + URL_FILE:=$(NETIFYD_FILE_PROC_FLOW_ACTIONS) + FILE:=$(NETIFYD_FILE_PROC_FLOW_ACTIONS) + HASH:=$(NETIFYD_HASH_PROC_FLOW_ACTIONS) endef $(eval $(call Download,libnetify-proc-flow-actions)) define Download/libnetify-sink-http URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-sink-http.so.0.0.0 - FILE:=libnetify-sink-http.so.0.0.0 - HASH:=95d0d6fa2b47b3764318a2f8680441d126c94bb39c84440b3e056767e330991d + URL_FILE:=$(NETIFYD_FILE_SINK_HTTP) + FILE:=$(NETIFYD_FILE_SINK_HTTP) + HASH:=$(NETIFYD_HASH_SINK_HTTP) endef $(eval $(call Download,libnetify-sink-http)) define Download/libnetify-sink-log URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-sink-log.so.0.0.0 - FILE:=libnetify-sink-log.so.0.0.0 - HASH:=f9cafc30e150109dc3ab5e57eb203d51525a4ad0241fe76724206edf73c575b1 + URL_FILE:=$(NETIFYD_FILE_SINK_LOG) + FILE:=$(NETIFYD_FILE_SINK_LOG) + HASH:=$(NETIFYD_HASH_SINK_LOG) endef $(eval $(call Download,libnetify-sink-log)) -define Download/libnetify-sink-socket - URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-sink-socket.so.0.0.0 - FILE:=libnetify-sink-socket.so.0.0.0 - HASH:=7ed9fe8d4d952ccbedca1c2be505fe288080f19c5c60d4db46932078c97d5364 -endef -$(eval $(call Download,libnetify-sink-socket)) - -define Download/libnetify-sink-sqlite - URL:=$(NETIFYD_BASE_URL)/usr/lib - URL_FILE:=libnetify-sink-sqlite.so.0.0.0 - FILE:=libnetify-sink-sqlite.so.0.0.0 - HASH:=0de0ef72cbd092fedd5cc278a4b57977891650aa10712f5ae0e6381a5a02fa52 -endef -$(eval $(call Download,libnetify-sink-sqlite)) - define Download/netifyd URL:=$(NETIFYD_BASE_URL)/usr/sbin - URL_FILE:=netifyd - FILE:=netifyd - HASH:=a1d8f40f87ba58876c7652dc0f82e699d6f8139793b8ede9d33cea043a693c72 + URL_FILE:=$(NETIFYD_FILE_NETIFYD) + FILE:=$(NETIFYD_FILE_NETIFYD) + HASH:=$(NETIFYD_HASH_NETIFYD) endef $(eval $(call Download,netifyd)) define Build/Prepare mkdir -p $(PKG_BUILD_DIR) - $(CP) $(DL_DIR)/libnetifyd.so.4.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-plm.so.1.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-proc-aggregator.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-proc-core.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-proc-dev-discovery.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-proc-flow-actions.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-sink-http.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-sink-log.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-sink-socket.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/libnetify-sink-sqlite.so.0.0.0 $(PKG_BUILD_DIR)/ - $(CP) $(DL_DIR)/netifyd $(PKG_BUILD_DIR)/netifyd + $(CP) $(DL_DIR)/$(NETIFYD_FILE_LIBNETIFYD) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_PLM) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_PROC_AGGREGATOR) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_PROC_CORE) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_PROC_FLOW_ACTIONS) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_SINK_HTTP) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_SINK_LOG) $(PKG_BUILD_DIR)/ + $(CP) $(DL_DIR)/$(NETIFYD_FILE_NETIFYD) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_NETIFYD) endef define Build/Configure @@ -188,18 +164,16 @@ define Package/netifyd/install $(INSTALL_DIR) $(1)/etc/config $(INSTALL_DIR) $(1)/etc/init.d $(INSTALL_DIR) $(1)/etc/netifyd + $(INSTALL_DIR) $(1)/etc/netifyd/address-groups.d $(INSTALL_DIR) $(1)/etc/netifyd/categories.d $(INSTALL_DIR) $(1)/etc/netifyd/interfaces.d $(INSTALL_DIR) $(1)/etc/netifyd/plugins.d $(INSTALL_DIR) $(1)/etc/netifyd/profiles.d $(INSTALL_DIR) $(1)/etc/uci-defaults - $(INSTALL_DIR) $(1)/usr $(INSTALL_DIR) $(1)/usr/lib $(INSTALL_DIR) $(1)/usr/sbin - $(INSTALL_DIR) $(1)/usr/share $(INSTALL_DIR) $(1)/usr/share/netifyd $(INSTALL_DIR) $(1)/usr/share/netifyd/plugins.d - $(INSTALL_DIR) $(1)/usr/share/nftables.d/table-pre # netifyd $(INSTALL_DATA) ./files/etc/config/netifyd $(1)/etc/config/netifyd @@ -213,10 +187,10 @@ define Package/netifyd/install $(INSTALL_BIN) ./files/etc/uci-defaults/99-netify-enable-nfqueue.uci-default $(1)/etc/uci-defaults/99-netify-enable-nfqueue $(INSTALL_BIN) ./files/etc/uci-defaults/99-netify-disable-autoconfig.uci-default $(1)/etc/uci-defaults/99-netify-disable-autoconfig $(INSTALL_BIN) ./files/etc/uci-defaults/99-netify-disable-coredumps.uci-default $(1)/etc/uci-defaults/99-netify-disable-coredumps - $(INSTALL_BIN) $(PKG_BUILD_DIR)/netifyd $(1)/usr/sbin/netifyd - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetifyd.so.4.0.0 $(1)/usr/lib/libnetifyd.so.4.0.0 - $(LN) /usr/lib/libnetifyd.so.4.0.0 $(1)/usr/lib/libnetifyd.so - $(LN) /usr/lib/libnetifyd.so.4.0.0 $(1)/usr/lib/libnetifyd.so.4 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_NETIFYD) $(1)/usr/sbin/netifyd + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_LIBNETIFYD) $(1)/usr/lib/$(NETIFYD_FILE_LIBNETIFYD) + $(LN) /usr/lib/$(NETIFYD_FILE_LIBNETIFYD) $(1)/usr/lib/$(NETIFYD_LINK_LIBNETIFYD) + $(LN) /usr/lib/$(NETIFYD_FILE_LIBNETIFYD) $(1)/usr/lib/$(NETIFYD_SONAME_LIBNETIFYD) $(INSTALL_DATA) ./files/usr/share/netifyd/functions.sh $(1)/usr/share/netifyd/functions.sh $(INSTALL_DATA) ./files/usr/share/netifyd/netify-apps.conf $(1)/usr/share/netifyd/netify-apps.conf $(INSTALL_DATA) ./files/usr/share/netifyd/netify-categories.json $(1)/usr/share/netifyd/netify-categories.json @@ -227,57 +201,39 @@ define Package/netifyd/install # nethesis additional files $(INSTALL_BIN) ./files/usr/sbin/ns-netifyd-configure.py $(1)/usr/sbin/ns-netifyd-configure # netify-plm - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-plm.so.1.0.0 $(1)/usr/lib/libnetify-plm.so.1.0.0 - $(LN) /usr/lib/libnetify-plm.so.1.0.0 $(1)/usr/lib/libnetify-plm.so - $(LN) /usr/lib/libnetify-plm.so.1.0.0 $(1)/usr/lib/libnetify-plm.so.1 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_PLM) $(1)/usr/lib/$(NETIFYD_FILE_PLM) + $(LN) /usr/lib/$(NETIFYD_FILE_PLM) $(1)/usr/lib/$(NETIFYD_LINK_PLM) + $(LN) /usr/lib/$(NETIFYD_FILE_PLM) $(1)/usr/lib/$(NETIFYD_SONAME_PLM) # netify-proc-aggregator $(INSTALL_DATA) ./files/etc/netifyd/netify-proc-aggregator.json $(1)/etc/netifyd/netify-proc-aggregator.json $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-proc-aggregator.conf $(1)/etc/netifyd/plugins.d/10-netify-proc-aggregator.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-proc-aggregator.so.0.0.0 $(1)/usr/lib/libnetify-proc-aggregator.so.0.0.0 - $(LN) /usr/lib/libnetify-proc-aggregator.so.0.0.0 $(1)/usr/lib/libnetify-proc-aggregator.so - $(LN) /usr/lib/libnetify-proc-aggregator.so.0.0.0 $(1)/usr/lib/libnetify-proc-aggregator.so.0 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_PROC_AGGREGATOR) $(1)/usr/lib/$(NETIFYD_FILE_PROC_AGGREGATOR) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_AGGREGATOR) $(1)/usr/lib/$(NETIFYD_LINK_PROC_AGGREGATOR) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_AGGREGATOR) $(1)/usr/lib/$(NETIFYD_SONAME_PROC_AGGREGATOR) # netify-proc-core $(INSTALL_DATA) ./files/etc/netifyd/netify-proc-core.json $(1)/etc/netifyd/netify-proc-core.json $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-proc-core.conf $(1)/etc/netifyd/plugins.d/10-netify-proc-core.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-proc-core.so.0.0.0 $(1)/usr/lib/libnetify-proc-core.so.0.0.0 - $(LN) /usr/lib/libnetify-proc-core.so.0.0.0 $(1)/usr/lib/libnetify-proc-core.so - $(LN) /usr/lib/libnetify-proc-core.so.0.0.0 $(1)/usr/lib/libnetify-proc-core.so.0 - # netify-proc-dev-discovery - $(INSTALL_DATA) ./files/etc/netifyd/netify-proc-dev-discovery.json $(1)/etc/netifyd/netify-proc-dev-discovery.json - $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-proc-dev-discovery.conf $(1)/etc/netifyd/plugins.d/10-netify-proc-dev-discovery.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-proc-dev-discovery.so.0.0.0 $(1)/usr/lib/libnetify-proc-dev-discovery.so.0.0.0 - $(LN) /usr/lib/libnetify-proc-dev-discovery.so.0.0.0 $(1)/usr/lib/libnetify-proc-dev-discovery.so - $(LN) /usr/lib/libnetify-proc-dev-discovery.so.0.0.0 $(1)/usr/lib/libnetify-proc-dev-discovery.so.0 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_PROC_CORE) $(1)/usr/lib/$(NETIFYD_FILE_PROC_CORE) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_CORE) $(1)/usr/lib/$(NETIFYD_LINK_PROC_CORE) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_CORE) $(1)/usr/lib/$(NETIFYD_SONAME_PROC_CORE) # netify-proc-flow-actions $(INSTALL_DATA) ./files/etc/netifyd/netify-proc-flow-actions.json $(1)/etc/netifyd/netify-proc-flow-actions.json $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-proc-flow-actions.conf $(1)/etc/netifyd/plugins.d/10-netify-proc-flow-actions.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-proc-flow-actions.so.0.0.0 $(1)/usr/lib/libnetify-proc-flow-actions.so.0.0.0 - $(LN) /usr/lib/libnetify-proc-flow-actions.so.0.0.0 $(1)/usr/lib/libnetify-proc-flow-actions.so - $(LN) /usr/lib/libnetify-proc-flow-actions.so.0.0.0 $(1)/usr/lib/libnetify-proc-flow-actions.so.0 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_PROC_FLOW_ACTIONS) $(1)/usr/lib/$(NETIFYD_FILE_PROC_FLOW_ACTIONS) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_FLOW_ACTIONS) $(1)/usr/lib/$(NETIFYD_LINK_PROC_FLOW_ACTIONS) + $(LN) /usr/lib/$(NETIFYD_FILE_PROC_FLOW_ACTIONS) $(1)/usr/lib/$(NETIFYD_SONAME_PROC_FLOW_ACTIONS) # netify-sink-log $(INSTALL_DATA) ./files/etc/netifyd/netify-sink-log.json $(1)/etc/netifyd/netify-sink-log.json $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-sink-log.conf $(1)/etc/netifyd/plugins.d/10-netify-sink-log.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-sink-log.so.0.0.0 $(1)/usr/lib/libnetify-sink-log.so.0.0.0 - $(LN) /usr/lib/libnetify-sink-log.so.0.0.0 $(1)/usr/lib/libnetify-sink-log.so - $(LN) /usr/lib/libnetify-sink-log.so.0.0.0 $(1)/usr/lib/libnetify-sink-log.so.0 - # netify-sink-socket - $(INSTALL_DATA) ./files/etc/netifyd/netify-sink-socket.json $(1)/etc/netifyd/netify-sink-socket.json - $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-sink-socket.conf $(1)/etc/netifyd/plugins.d/10-netify-sink-socket.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-sink-socket.so.0.0.0 $(1)/usr/lib/libnetify-sink-socket.so.0.0.0 - $(LN) /usr/lib/libnetify-sink-socket.so.0.0.0 $(1)/usr/lib/libnetify-sink-socket.so - $(LN) /usr/lib/libnetify-sink-socket.so.0.0.0 $(1)/usr/lib/libnetify-sink-socket.so.0 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_SINK_LOG) $(1)/usr/lib/$(NETIFYD_FILE_SINK_LOG) + $(LN) /usr/lib/$(NETIFYD_FILE_SINK_LOG) $(1)/usr/lib/$(NETIFYD_LINK_SINK_LOG) + $(LN) /usr/lib/$(NETIFYD_FILE_SINK_LOG) $(1)/usr/lib/$(NETIFYD_SONAME_SINK_LOG) # netify-sink-http $(INSTALL_DATA) ./files/etc/netifyd/netify-sink-http.json $(1)/etc/netifyd/netify-sink-http.json $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-sink-http.conf $(1)/etc/netifyd/plugins.d/10-netify-sink-http.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-sink-http.so.0.0.0 $(1)/usr/lib/libnetify-sink-http.so.0.0.0 - $(LN) /usr/lib/libnetify-sink-http.so.0.0.0 $(1)/usr/lib/libnetify-sink-http.so - $(LN) /usr/lib/libnetify-sink-http.so.0.0.0 $(1)/usr/lib/libnetify-sink-http.so.0 - # netify-sink-sqlite - $(INSTALL_DATA) ./files/etc/netifyd/netify-sink-sqlite.json $(1)/etc/netifyd/netify-sink-sqlite.json - $(INSTALL_DATA) ./files/etc/netifyd/plugins.d/10-netify-sink-sqlite.conf $(1)/etc/netifyd/plugins.d/10-netify-sink-sqlite.conf - $(INSTALL_BIN) $(PKG_BUILD_DIR)/libnetify-sink-sqlite.so.0.0.0 $(1)/usr/lib/libnetify-sink-sqlite.so.0.0.0 - $(LN) /usr/lib/libnetify-sink-sqlite.so.0.0.0 $(1)/usr/lib/libnetify-sink-sqlite.so - $(LN) /usr/lib/libnetify-sink-sqlite.so.0.0.0 $(1)/usr/lib/libnetify-sink-sqlite.so.0 + $(INSTALL_BIN) $(PKG_BUILD_DIR)/$(NETIFYD_FILE_SINK_HTTP) $(1)/usr/lib/$(NETIFYD_FILE_SINK_HTTP) + $(LN) /usr/lib/$(NETIFYD_FILE_SINK_HTTP) $(1)/usr/lib/$(NETIFYD_LINK_SINK_HTTP) + $(LN) /usr/lib/$(NETIFYD_FILE_SINK_HTTP) $(1)/usr/lib/$(NETIFYD_SONAME_SINK_HTTP) endef $(eval $(call BuildPackage,netifyd)) diff --git a/packages/netifyd/files/etc/netifyd.conf b/packages/netifyd/files/etc/netifyd.conf index 0ee170965..dd6b37370 100644 --- a/packages/netifyd/files/etc/netifyd.conf +++ b/packages/netifyd/files/etc/netifyd.conf @@ -32,7 +32,4 @@ path_license_manager = ${path_plugin_libdir}/libnetify-plm.so # command-line parameters. auto_informatics = no -# disabling coredumps -enable_coredumps = no - # vim: set ft=dosini : diff --git a/packages/netifyd/files/etc/netifyd/netify-proc-core.json b/packages/netifyd/files/etc/netifyd/netify-proc-core.json index 6966afe2f..bd7715df3 100644 --- a/packages/netifyd/files/etc/netifyd/netify-proc-core.json +++ b/packages/netifyd/files/etc/netifyd/netify-proc-core.json @@ -1,5 +1,24 @@ { "format": "json", "compressor": "none", - "sinks": { } -} \ No newline at end of file + "sinks": { + "sink-http": { + "legacy": { + "enable": false, + "types": [ "legacy-http" ], + "format": "json", + "compressor": "gz" + } + }, + "sink-mqtt": { + "flows": { + "enable": false, + "types": [ "stream-flows" ] + }, + "stats": { + "enable": false, + "types": [ "stream-stats" ] + } + } + } +} diff --git a/packages/netifyd/files/etc/netifyd/netify-proc-dev-discovery.json b/packages/netifyd/files/etc/netifyd/netify-proc-dev-discovery.json deleted file mode 100644 index aab0f12fe..000000000 --- a/packages/netifyd/files/etc/netifyd/netify-proc-dev-discovery.json +++ /dev/null @@ -1,45 +0,0 @@ -{ - "compressor": "gz", - "format": "json", - - "max_confidence": 80, - "max_devices": 1500, - "max_device_age": 4147200, - "max_http_user_agents": { - "max": 10, - "update_min": 10, - "update_ttl": 3600 - }, - "max_ja4_clients": 10, - "max_mdns_services": 10, - "max_ssdp_user_agents": 10, - "process_all_macs": false, - "path_device_cache": "${path_state_persistent}/device-discovery-cache.json", - "device_mac_ignore": [ - "00:00:00:00:00:00", - "ff:ff:ff:ff:ff:ff" - ], - "device_mac_ignore_group": "dev-disc-ignore", - "netify_api": { - "enable": false, - "url": "https://agents.netify.ai/api/v2/device_discovery", - "key": "deadbeef-0000-ffff-0000-0123456789ff" - }, - "ubus_api": { - "enable": false, - "sink": "sink-ubus", - "sink_channel": "device_discovery", - "ubus_subscribe": "device_discovery", - "format": "json" - }, - "sinks": { - "sink-log": { - "default": { - "enable": false, - "flush": false, - "format": "json", - "compressor": "gz" - } - } - } -} diff --git a/packages/netifyd/files/etc/netifyd/netify-sink-http.json b/packages/netifyd/files/etc/netifyd/netify-sink-http.json index 9e975c20b..48c8ffccd 100644 --- a/packages/netifyd/files/etc/netifyd/netify-sink-http.json +++ b/packages/netifyd/files/etc/netifyd/netify-sink-http.json @@ -3,5 +3,18 @@ "timeout_transfer": 300, "tls_verify": true, "tls_version1": false, - "channels": { } + "channels": { + "legacy": { + "enable": false, + "timeout_connect": 30, + "timeout_transfer": 300, + "url": "https://sink.netify.ai/v1/", + "headers": { + "x-vendor-id": "EG", + "X-UUID": "${uuid_agent}", + "X-UUID-Site": "${uuid_site}", + "X-UUID-Serial": "${uuid_serial}" + } + } + } } diff --git a/packages/netifyd/files/etc/netifyd/netify-sink-socket.json b/packages/netifyd/files/etc/netifyd/netify-sink-socket.json deleted file mode 100644 index df67b182b..000000000 --- a/packages/netifyd/files/etc/netifyd/netify-sink-socket.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "channels": { } -} \ No newline at end of file diff --git a/packages/netifyd/files/etc/netifyd/netify-sink-sqlite.json b/packages/netifyd/files/etc/netifyd/netify-sink-sqlite.json deleted file mode 100644 index 1bbe92973..000000000 --- a/packages/netifyd/files/etc/netifyd/netify-sink-sqlite.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "db_path": "${path_state_persistent}/db", - "purge_in_days": 30, - "channels": { - "default": { - "enable": true, - "db_name": "stats.db", - "data_source": "aggregator", - "purge_in_days": 14, - "tables": [ - { - "name": "stats", - "columns": [ - { - "name": "application_id", - "mapped": "detected_application", - "type": "INTEGER" - }, - { - "name": "protocol_id", - "mapped": "detected_protocol", - "type": "INTEGER" - }, - { - "name": "application_category_id", - "mapped": "application_category", - "type": "INTEGER" - }, - { - "name": "protocol_category_id", - "mapped": "protocol_category", - "type": "INTEGER" - }, - { - "name": "overlay_tags", - "mapped": "tags", - "type": "TEXT" - }, - { - "name": "tag", - "mapped": "tags", - "split": 0, - "type": "TEXT" - }, - { - "name": "tag_group", - "mapped": "tags", - "split": 1, - "type": "TEXT" - }, - { - "name": "download_bytes", - "mapped": "other_bytes", - "type": "INTEGER", - "default": 0 - }, - { - "name": "download_packets", - "mapped": "other_packets", - "type": "INTEGER", - "default": 0 - }, - { - "name": "upload_bytes", - "mapped": "local_bytes", - "type": "INTEGER", - "default": 0 - }, - { - "name": "upload_packets", - "mapped": "local_packets", - "type": "INTEGER", - "default": 0 - }, - { - "name": "local_ip", - "mapped": "local_ip", - "type": "TEXT" - }, - { - "name": "local_mac", - "mapped": "local_mac", - "type": "TEXT" - } - ] - } - ] - } - } -} diff --git a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-proc-dev-discovery.conf b/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-proc-dev-discovery.conf deleted file mode 100644 index 556c06200..000000000 --- a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-proc-dev-discovery.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Netify Device Discovery Processor Plugin Loader -# -############################################################################## - -[proc-dev-discovery] -enable = no -plugin_library = ${path_plugin_libdir}/libnetify-proc-dev-discovery.so.0.0.0 -conf_filename = ${path_state_persistent}/netify-proc-dev-discovery.json - -# vim: set ft=dosini : diff --git a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-http.conf b/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-http.conf index 570971be1..4d8ab5156 100644 --- a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-http.conf +++ b/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-http.conf @@ -3,7 +3,7 @@ ############################################################################## [sink-http] -enable = yes +enable = no plugin_library = ${path_plugin_libdir}/libnetify-sink-http.so.0.0.0 conf_filename = ${path_state_persistent}/netify-sink-http.json diff --git a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-socket.conf b/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-socket.conf deleted file mode 100644 index 516fb6278..000000000 --- a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-socket.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Netify Socket Sink Plugin Loader -# -############################################################################## - -[sink-socket] -enable = no -plugin_library = ${path_plugin_libdir}/libnetify-sink-socket.so.0.0.0 -conf_filename = ${path_state_persistent}/netify-sink-socket.json - -# vim: set ft=dosini : diff --git a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-sqlite.conf b/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-sqlite.conf deleted file mode 100644 index 13005c1ac..000000000 --- a/packages/netifyd/files/etc/netifyd/plugins.d/10-netify-sink-sqlite.conf +++ /dev/null @@ -1,10 +0,0 @@ -# Netify SQLite Sink Plugin Loader -# -############################################################################## - -[sink-sqlite] -enable = no -plugin_library = ${path_plugin_libdir}/libnetify-sink-sqlite.so.0.0.0 -conf_filename = ${path_state_persistent}/netify-sink-sqlite.json - -# vim: set ft=dosini : diff --git a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf index 27afecf22..808c48703 100644 --- a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf +++ b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf @@ -81,7 +81,7 @@ use_getifaddrs = false # Local command/control socket. Receive commands and control messages on a # local UNIX socket. -#path_server_socket = ${path_state_volatile}/netifyd.sock +path_server_socket = ${path_state_volatile}/netifyd.sock # Capture Defaults ############################################################################ @@ -139,11 +139,15 @@ private_external_addresses = no ############################################################################ [netify-api] -# Enable/disable API integration with Netify Informatics +# Enable/disable API for app signatures, intel signatures, and more... enable = no update_tick = 30 update_interval = 86400 tls_verify = yes +#mtls_enable = no +#ca_file = /path/to/ca.pem +#cert_file = /path/to/cert.pem +#key_file = /path/to/private.key # Protocol Dissector Options ############################################################################ @@ -156,8 +160,62 @@ all = include ############################################################################ [netlink] -# Set the Netlink buffer size -buffer_size = 327680 -bridge_pvid_discovery = no +# Set the user-space buffer size for Netlink messages. +buffer_size = 65536 +# Set the maximum size of the Netlink buffer size: +# If you see warnings such as: mnl_socket_recvfrom: No buffer space available +# There are two options; adjust the buffer_size above by increasing it until +# the warnings stop, or set the max_buffer_size to either 0 or some higher +# value such as double or triple the buffer_size. Then when the buffer is +# full, it will automatically increased by 2 KB until it reaches the +# max_buffer_size. If max_buffer_size is 0, it will increase indefinately +# until the warnings stop. To disable automatic buffer resizing, set the +# max_buffer_size to the same value as buffer_size, ex: +# buffer_size = 65535 +# max_buffer_size = 65535 +# Nethesis patch: x5 the buffer_size +max_buffer_size = 327680 +# Set the buffer size (1MB default, 8MB for higher traffic spikes) +# for Netlink sockets. Set to 0 to use the kernel's default. +socket_buffer_size = 1048576 +# In the event we miss a conntrack DESTROY event, clean-up cached flow +# entries that have been idle for greater than this number of seconds. +# Default: 1 hour (3600). Set to 0 to disable. +conntrack_idle_timeout = 3600 + +# Enhanced interface metadata using Socket Buffer Marks +############################################################################ + +[mark-to-interface] +# Enables the logical mapping of packets to their original source +# interface. When enabled, the agent extracts a stored interface index from the +# packet mark (skb mark) to identify the specific logical interface (e.g., a +# wireless VAP) even when packets are captured on a physical bridge or NFQueue handle. +# enable = no + +# Specifies the bitwise left-shift used to locate the interface index within the +# 32-bit packet mark. For example, a shift of 8 indicates the index is stored in +# the second byte (bits 8-15), allowing the mark to coexist with other metadata +# like connection marks. +# mark_shift = 8 + +# Specifies the bitwise mask used to locate the interface index within the +# 32-bit packet mark. +# mark_shift = 8 + +# Defines the firewall backend used to manage the mapping. When set to nft +# (currently the only mode supported), the agent will automatically synchronize +# the interface-to-mark mappings with the kernel's nftables sets. +# firewall = nft + +# Specifies the name of the nftables table where the mapping set is located. The +# agent expects to find (or will attempt to update) a set named according the the +# nft_set configuration parameter within this table to facilitate the correlation +# between physical and logical interfaces. +# nft_table = netify + +# A set within the nft_table used to facilitate the correlation between physical +# and logical interfaces. +# nft_set = ifindex_to_mark # vim: set ft=dosini : diff --git a/packages/netifyd/netify-dist.mk b/packages/netifyd/netify-dist.mk new file mode 100644 index 000000000..aee16116e --- /dev/null +++ b/packages/netifyd/netify-dist.mk @@ -0,0 +1,95 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# generated by tools/netifyd-update/netifyd-update.py, do not edit by hand + +ifeq ($(ARCH),x86_64) +NETIFYD_FILE_NETIFYD:=netifyd +NETIFYD_HASH_NETIFYD:=6e7eab582e50318312fd63e3513beb8c11e5ccae86a92f071ba665f5ff14b009 +NETIFYD_FILE_LIBNETIFYD:=libnetifyd.so.6.0.0 +NETIFYD_SONAME_LIBNETIFYD:=libnetifyd.so.6 +NETIFYD_LINK_LIBNETIFYD:=libnetifyd.so +NETIFYD_HASH_LIBNETIFYD:=c0241ccc9b80ed71af0456fba435b1f1dd77b6807297b0937ca021df1d6f4d32 +NETIFYD_FILE_PLM:=libnetify-plm.so.1.0.0 +NETIFYD_SONAME_PLM:=libnetify-plm.so.1 +NETIFYD_LINK_PLM:=libnetify-plm.so +NETIFYD_HASH_PLM:=fdd7cc484a747125930e70a426ae714509888b34ab3f3dd3df70b421aff07a5e +NETIFYD_FILE_PROC_AGGREGATOR:=libnetify-proc-aggregator.so.0.0.0 +NETIFYD_SONAME_PROC_AGGREGATOR:=libnetify-proc-aggregator.so.0 +NETIFYD_LINK_PROC_AGGREGATOR:=libnetify-proc-aggregator.so +NETIFYD_HASH_PROC_AGGREGATOR:=49e19c6f827e70a50ad71f8d2cd55ddefeb08c7976184c69a4a7ec03e50b17cf +NETIFYD_FILE_PROC_CORE:=libnetify-proc-core.so.0.0.0 +NETIFYD_SONAME_PROC_CORE:=libnetify-proc-core.so.0 +NETIFYD_LINK_PROC_CORE:=libnetify-proc-core.so +NETIFYD_HASH_PROC_CORE:=7c3c661857e46ed4aed16da6f2f97999ab02a0b805c535cb0df6126899f2fa69 +NETIFYD_FILE_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so.0.0.0 +NETIFYD_SONAME_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so.0 +NETIFYD_LINK_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so +NETIFYD_HASH_PROC_FLOW_ACTIONS:=69d36190737f8c739f8e7ee84810a833c38b0f0335db0094ca1cf2d54ad6dc00 +NETIFYD_FILE_SINK_HTTP:=libnetify-sink-http.so.0.0.0 +NETIFYD_SONAME_SINK_HTTP:=libnetify-sink-http.so.0 +NETIFYD_LINK_SINK_HTTP:=libnetify-sink-http.so +NETIFYD_HASH_SINK_HTTP:=4b7f3f76d38c556c4fc1965a0458f8478eb7c872547248f46c5c5c0a143fccf0 +NETIFYD_FILE_SINK_LOG:=libnetify-sink-log.so.0.0.0 +NETIFYD_SONAME_SINK_LOG:=libnetify-sink-log.so.0 +NETIFYD_LINK_SINK_LOG:=libnetify-sink-log.so +NETIFYD_HASH_SINK_LOG:=3ac13fd2b4b0ef035df529c5998a9d833e2a41372d20923885057f1e14e5a167 +else ifeq ($(ARCH),aarch64) +NETIFYD_FILE_NETIFYD:=netifyd +NETIFYD_HASH_NETIFYD:=3cf1920be81a756b1bd8ff89855c35d00e9e069b6cf8d65c62f63a6e6f3ca27e +NETIFYD_FILE_LIBNETIFYD:=libnetifyd.so.15.0.0 +NETIFYD_SONAME_LIBNETIFYD:=libnetifyd.so.15 +NETIFYD_LINK_LIBNETIFYD:=libnetifyd.so +NETIFYD_HASH_LIBNETIFYD:=9ee8c3e3d0cded536c2759edf500ea0fff1dfddc79743b5a6bf79b439041d48c +NETIFYD_FILE_PLM:=libnetify-plm.so.1.0.0 +NETIFYD_SONAME_PLM:=libnetify-plm.so.1 +NETIFYD_LINK_PLM:=libnetify-plm.so +NETIFYD_HASH_PLM:=6e1cc96df0eb88bd4982f2716e21f894f309c5725024934be839ebc3fa9d5fc2 +NETIFYD_FILE_PROC_AGGREGATOR:=libnetify-proc-aggregator.so.0.0.0 +NETIFYD_SONAME_PROC_AGGREGATOR:=libnetify-proc-aggregator.so.0 +NETIFYD_LINK_PROC_AGGREGATOR:=libnetify-proc-aggregator.so +NETIFYD_HASH_PROC_AGGREGATOR:=6c3e8612446bb7f73fff1abc969ab4f3abded6f1ffb5bd9bd41907ce154a0ee9 +NETIFYD_FILE_PROC_CORE:=libnetify-proc-core.so.0.0.0 +NETIFYD_SONAME_PROC_CORE:=libnetify-proc-core.so.0 +NETIFYD_LINK_PROC_CORE:=libnetify-proc-core.so +NETIFYD_HASH_PROC_CORE:=30543afd01d30ddb3501e2d35b4c2ef8932649241d6d79052de70eafd9e7905a +NETIFYD_FILE_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so.0.0.0 +NETIFYD_SONAME_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so.0 +NETIFYD_LINK_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so +NETIFYD_HASH_PROC_FLOW_ACTIONS:=98331ecd54ee05c7155a4072ec7ceeaa9430a6231fe09ef67f8d4c03d446f9ac +NETIFYD_FILE_SINK_HTTP:=libnetify-sink-http.so.0.0.0 +NETIFYD_SONAME_SINK_HTTP:=libnetify-sink-http.so.0 +NETIFYD_LINK_SINK_HTTP:=libnetify-sink-http.so +NETIFYD_HASH_SINK_HTTP:=60b884abc641308c66207bf7de348b4b3f8c2d4b2d0091f3879dfa4f31a49fd8 +NETIFYD_FILE_SINK_LOG:=libnetify-sink-log.so.0.0.0 +NETIFYD_SONAME_SINK_LOG:=libnetify-sink-log.so.0 +NETIFYD_LINK_SINK_LOG:=libnetify-sink-log.so +NETIFYD_HASH_SINK_LOG:=4dab9d43bda317f7c19269cd675ff8640417faeb82211a599abf8a1ddf2110e2 +else +# metadata scan (DUMP=1, no .config) or an arch we ship no binaries for: +# placeholders only, netifyd is not selectable there and nothing is downloaded +NETIFYD_FILE_NETIFYD:=netifyd +NETIFYD_FILE_LIBNETIFYD:=libnetifyd.so +NETIFYD_SONAME_LIBNETIFYD:=libnetifyd.so +NETIFYD_LINK_LIBNETIFYD:=libnetifyd.so +NETIFYD_FILE_PLM:=libnetify-plm.so +NETIFYD_SONAME_PLM:=libnetify-plm.so +NETIFYD_LINK_PLM:=libnetify-plm.so +NETIFYD_FILE_PROC_AGGREGATOR:=libnetify-proc-aggregator.so +NETIFYD_SONAME_PROC_AGGREGATOR:=libnetify-proc-aggregator.so +NETIFYD_LINK_PROC_AGGREGATOR:=libnetify-proc-aggregator.so +NETIFYD_FILE_PROC_CORE:=libnetify-proc-core.so +NETIFYD_SONAME_PROC_CORE:=libnetify-proc-core.so +NETIFYD_LINK_PROC_CORE:=libnetify-proc-core.so +NETIFYD_FILE_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so +NETIFYD_SONAME_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so +NETIFYD_LINK_PROC_FLOW_ACTIONS:=libnetify-proc-flow-actions.so +NETIFYD_FILE_SINK_HTTP:=libnetify-sink-http.so +NETIFYD_SONAME_SINK_HTTP:=libnetify-sink-http.so +NETIFYD_LINK_SINK_HTTP:=libnetify-sink-http.so +NETIFYD_FILE_SINK_LOG:=libnetify-sink-log.so +NETIFYD_SONAME_SINK_LOG:=libnetify-sink-log.so +NETIFYD_LINK_SINK_LOG:=libnetify-sink-log.so +endif diff --git a/packages/ns-dpi/files/dpi-update.py b/packages/ns-dpi/files/dpi-update.py index 253f254e4..cb3b5484c 100755 --- a/packages/ns-dpi/files/dpi-update.py +++ b/packages/ns-dpi/files/dpi-update.py @@ -13,6 +13,8 @@ import subprocess import logging from os import environ +import semver +import json SUBSCRIPTION_SERVER = "https://sp.nethesis.it" @@ -24,11 +26,9 @@ def get_netifyd_version() -> str: try: result = subprocess.run( - ["netifyd", "--version"], capture_output=True, text=True + ["apk", "query", "--format", "json", "--field", "version", "netifyd"], capture_output=True, text=True ) - for line in (result.stdout + result.stderr).splitlines(): - if "Netify Agent/" in line: - return line.split("/")[1].split(" ")[0] + return json.loads(result.stdout.strip())[0]["version"].split("-")[0] except Exception as e: logging.warning(f"Failed to get netifyd version: {e}") return "" diff --git a/scripts/Readme.md b/scripts/Readme.md index 32fa85a37..2e25dff70 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -96,30 +96,3 @@ This script retrieves open issues labeled "testing" from the NethServer/nethsecu - 0 - Success - 1 - Error when sending the message or missing Mattermost webhook URL - 2 - Error when loading issues from GitHub - -## netifyd-packages.sh - -This script extracts Netify `.apk` packages from the `netifyd-apks` directory, unpacks them, and merges the files for analysis and integration. This script is useful when an update of `netifyd` requires changes in the integration meta package. - -### Prerequisites - -- **apk-tools**: The Alpine package extraction tool must be installed on your system. - - On Fedora/RHEL: `dnf install apk-tools` - - On Debian/Ubuntu: `apt-get install apk-tools` - - On Alpine Linux: Already included - -### Usage - -```bash -./netifyd-packages.sh -``` - -The script will process all `.apk` files in the `netifyd-apks/{arch}/` directories and extract their contents into `netifyd-apks/tmp/{arch}/netifyd/`. The merged files can then be copied into the `packages/netifyd/` directory as needed. - -### How It Works - -1. Iterates over each architecture subdirectory in `netifyd-apks/` -2. For each `.apk` file found, extracts its contents to a temporary location -3. Merges all extracted files into a single `netifyd` output directory per architecture -4. Skips metadata files (`.PKGINFO`, install scripts, etc.) during extraction -5. Outputs the merged file structure in `netifyd-apks/tmp/{arch}/netifyd/` diff --git a/scripts/netifyd-ipks/.gitignore b/scripts/netifyd-ipks/.gitignore deleted file mode 100644 index d6b7ef32c..000000000 --- a/scripts/netifyd-ipks/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/scripts/netifyd-packages.sh b/scripts/netifyd-packages.sh deleted file mode 100755 index d3261af1c..000000000 --- a/scripts/netifyd-packages.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -# This is a helper script to extract all Netify APK packages -# into netifyd-apks/tmp/{arch} directories for analysis and integration. -# -# Usage: ./netifyd-packages.sh -# -# Input: netifyd-apks/{arch}/*.apk -# Output: netifyd-apks/tmp/{arch}/netifyd/ — merged contents of all APKs for that arch -# netifyd-apks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) - -set -e - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -APKS_DIR="${SCRIPT_DIR}/netifyd-apks" -TMP_DIR="${APKS_DIR}/tmp" - -if [ ! -d "${APKS_DIR}" ]; then - echo "ERROR: APKs directory not found: ${APKS_DIR}" >&2 - exit 1 -fi - -# Iterate over each arch subdirectory -for arch_dir in "${APKS_DIR}"/*/; do - [ -d "${arch_dir}" ] || continue - - # Skip the tmp directory - [ "$(basename "${arch_dir}")" = "tmp" ] && continue - - arch="$(basename "${arch_dir}")" - - echo "==> Processing arch: ${arch}" - - output_dir="${TMP_DIR}/${arch}/netifyd" - rm -rf "${TMP_DIR:?}/${arch}" - mkdir -p "${output_dir}" - - # Extract each APK into its own per-package subdirectory, then merge - for apk_file in "${arch_dir}"*.apk; do - [ -f "${apk_file}" ] || continue - - # Derive package name: strip version and arch suffix - # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.apk -> netify-plm - filename="$(basename "${apk_file}" .apk)" - pkg_name="${filename%%_*}" - - # Extract to a temporary directory first to avoid conflicts - pkg_extract_dir="${TMP_DIR}/${arch}/.extract-${pkg_name}-$$" - mkdir -p "${pkg_extract_dir}" - - echo " Extracting ${filename} -> ${pkg_name}/" - apk extract --allow-untrusted --destination "${pkg_extract_dir}" "${apk_file}" - - # Merge extracted files into the single netifyd output directory - cp -a "${pkg_extract_dir}/." "${output_dir}/" - done - - echo " Merged output: tmp/${arch}/netifyd/" -done - -echo "Done." diff --git a/tools/README.md b/tools/README.md index 468e6f053..ebf10bca0 100644 --- a/tools/README.md +++ b/tools/README.md @@ -6,6 +6,7 @@ Tools: - cleanup: used by `cleanup.yml` to remove old image versions from the CDN - issue-comment: used by `build-image.yml` to comment on issues when a PR is merged into the main branch +- netifyd-update: refresh the netifyd binaries pinned by the `netifyd` package ## cleanup @@ -73,3 +74,30 @@ The script will: Generated changelos are saved in the current directory: - core-changes.md - packages-changes.md + +## netifyd-update + +Refresh the netifyd binaries used by the `netifyd` package. + +The script: +- reads the upstream index (`SOURCES` at the top of the script, one entry per architecture) +- downloads the apk of every package listed in `PACKAGES` +- extracts them with `apk extract` +- writes `packages/netifyd/netify-dist.mk` with file name, soname, unversioned link and + sha256 of each shipped binary, per architecture +- copies the real files (not the symlinks, they are recreated at install time) into + `tools/netifyd-update/dist//`, laid out as the Makefile downloads them + +Requirements: `apk-tools` (`dnf install apk-tools` or `apt-get install apk-tools`) and +the Python dependencies in `requirements.txt`. + +Usage example: +``` +pip install -r requirements.txt +tools/netifyd-update/netifyd-update.py update +``` + +Use `--force` to remove an existing `work/` and `dist/` from a previous run and redo it. + +The `dist/` tree must then be published manually to the netifyd-dist mirror, under +`netifyd-//`. diff --git a/tools/netifyd-update/netifyd-update.py b/tools/netifyd-update/netifyd-update.py new file mode 100755 index 000000000..5001ab27f --- /dev/null +++ b/tools/netifyd-update/netifyd-update.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +import argparse +import hashlib +import shutil +import subprocess +import sys +import requests +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DEFAULT_WORK_DIR = SCRIPT_DIR / "work" +DEFAULT_DIST_DIR = SCRIPT_DIR / "dist" +NETIFYD_PKG_DIR = SCRIPT_DIR.parent.parent / "packages" / "netifyd" +DIST_MK = NETIFYD_PKG_DIR / "netify-dist.mk" + +SOURCES: dict[str, str] = { + "x86": "https://download.netify.ai/5/openwrt/25.12/x86/index.json", +} + +# maps a SOURCES key to the OpenWrt $(ARCH) value used by the Makefile +MAKEFILE_ARCH: dict[str, str] = {"x86": "x86_64", "aarch64": "aarch64"} + +PACKAGES: list[str] = [ + "netify-plm", + "netify-proc-aggregator", + "netify-proc-core", + "netify-proc-flow-actions", + "netify-sink-http", + "netify-sink-log", + "netifyd", +] + +# maps an apk package to the extracted files whose name and sha256 feed the matching +# Makefile variables; netifyd ships both the binary and libnetifyd.so +DIST_TARGETS: dict[str, list[tuple[str, str]]] = { + "netifyd": [("usr/sbin/netifyd", "NETIFYD"), ("usr/lib/libnetifyd.so.*", "LIBNETIFYD")], + "netify-plm": [("usr/lib/libnetify-plm.so.*", "PLM")], + "netify-proc-aggregator": [("usr/lib/libnetify-proc-aggregator.so.*", "PROC_AGGREGATOR")], + "netify-proc-core": [("usr/lib/libnetify-proc-core.so.*", "PROC_CORE")], + "netify-proc-flow-actions": [("usr/lib/libnetify-proc-flow-actions.so.*", "PROC_FLOW_ACTIONS")], + "netify-sink-http": [("usr/lib/libnetify-sink-http.so.*", "SINK_HTTP")], + "netify-sink-log": [("usr/lib/libnetify-sink-log.so.*", "SINK_LOG")], +} + +# flattened, stable order of the variable suffixes for netify-dist.mk +DIST_SUFFIX_ORDER: list[str] = [suffix for targets in DIST_TARGETS.values() for _, suffix in targets] + +# collected field -> Makefile variable infix, in emission order +FIELD_ORDER: list[tuple[str, str]] = [ + ("file", "FILE"), + ("soname", "SONAME"), + ("link", "LINK"), + ("hash", "HASH"), +] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + update = subparsers.add_parser("update", help="download the upstream apks, regenerate netify-dist.mk and dist/") + update.add_argument("--force", action="store_true", help="remove existing work and dist directories and redo") + + return parser.parse_args() + + +def fail(message: str) -> None: + print(f"error: {message}", file=sys.stderr) + sys.exit(1) + + +def warn(message: str) -> None: + print(f"warning: {message}", file=sys.stderr) + + +def download_file(url: str, dest: Path) -> None: + with requests.get(url, stream=True) as response: + response.raise_for_status() + with open(dest, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + print(f"Downloaded {dest.name} to {dest.parent}") + + +def extract_apk(apk_path: Path, dest_dir: Path) -> None: + dest_dir.mkdir(parents=True, exist_ok=True) + print(f"Extracting {apk_path.name} to {dest_dir}") + subprocess.run(["apk", "extract", "--allow-untrusted", "--destination", str(dest_dir), str(apk_path)], check=True) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(8192), b""): + digest.update(chunk) + return digest.hexdigest() + + +def symlinks_to(target: Path) -> list[str]: + """Names of the sibling symlinks pointing at target, shortest first.""" + resolved = target.resolve() + names = [p.name for p in target.parent.iterdir() if p.is_symlink() and p.resolve() == resolved] + return sorted(names, key=len) + + +def collect_targets( + package: str, + extract_dir: Path, + dist_dir: Path, + makefile_arch: str, + entries: dict[str, dict[str, dict]], +) -> None: + for pattern, suffix in DIST_TARGETS.get(package, []): + matches = [p for p in extract_dir.glob(pattern) if not p.is_symlink()] + if len(matches) != 1: + fail(f"expected exactly one file matching {pattern} in {extract_dir}, found {len(matches)}") + + target = matches[0] + fields = {"file": target.name, "hash": sha256_file(target)} + + # shared objects ship an unversioned and a soname symlink, plain binaries ship none + links = symlinks_to(target) + if links: + if len(links) != 2: + fail(f"expected exactly two symlinks to {target.name} in {target.parent}, found {len(links)}") + fields["link"], fields["soname"] = links + + entries.setdefault(suffix, {})[makefile_arch] = fields + + # mirror the file under the layout the Makefile downloads from; the symlinks are + # recreated at install time, only the real files get published + published = dist_dir / target.relative_to(extract_dir) + published.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(target, published) + + +def warn_on_diverging_filenames(entries: dict[str, dict[str, dict]], arches: list[str]) -> None: + for suffix in DIST_SUFFIX_ORDER: + names = {arch: entries[suffix][arch]["file"] for arch in arches} + if len(set(names.values())) > 1: + detail = " ".join(f"{arch}={name}" for arch, name in names.items()) + warn(f"{suffix} filename differs across arches, upstream feeds are not aligned: {detail}") + + +def write_dist_mk(entries: dict[str, dict[str, dict]], arches: list[str]) -> None: + lines = [ + "#", + "# Copyright (C) 2026 Nethesis S.r.l.", + "# SPDX-License-Identifier: GPL-2.0-only", + "#", + "", + "# generated by tools/netifyd-update/netifyd-update.py, do not edit by hand", + "", + ] + for i, arch in enumerate(arches): + keyword = "ifeq" if i == 0 else "else ifeq" + lines.append(f"{keyword} ($(ARCH),{arch})") + for suffix in DIST_SUFFIX_ORDER: + fields = entries[suffix][arch] + for field, infix in FIELD_ORDER: + if field in fields: + lines.append(f"NETIFYD_{infix}_{suffix}:={fields[field]}") + # the remaining branch covers the metadata scan, which runs with DUMP=1 and therefore + # without .config, leaving $(ARCH) empty, and any arch the package is not selectable + # on; the Download macro rejects an empty FILE either way, so hand it the unversioned + # names as placeholders, nothing gets fetched in those cases + lines.append("else") + lines.append("# metadata scan (DUMP=1, no .config) or an arch we ship no binaries for:") + lines.append("# placeholders only, netifyd is not selectable there and nothing is downloaded") + for suffix in DIST_SUFFIX_ORDER: + fields = entries[suffix][arches[0]] + placeholder = fields.get("link", fields["file"]) + lines.append(f"NETIFYD_FILE_{suffix}:={placeholder}") + if "soname" in fields: + lines.append(f"NETIFYD_SONAME_{suffix}:={placeholder}") + lines.append(f"NETIFYD_LINK_{suffix}:={placeholder}") + lines.append("endif") + lines.append("") + + DIST_MK.write_text("\n".join(lines)) + print(f"Wrote {DIST_MK}") + + +def run_update(args: argparse.Namespace) -> None: + if shutil.which("apk") is None: + fail("apk-tools is required: install it with 'dnf install apk-tools' or 'apt-get install apk-tools'") + + # bail out if a previous run's directories are still there + for stale in (DEFAULT_WORK_DIR, DEFAULT_DIST_DIR): + if stale.exists(): + if not args.force: + fail(f"{stale} already exists, use --force to remove it and redo") + shutil.rmtree(stale) + + entries: dict[str, dict[str, dict]] = {} + makefile_arches: list[str] = [] + + for arch, index_url in SOURCES.items(): + makefile_arch = MAKEFILE_ARCH[arch] + makefile_arches.append(makefile_arch) + + # fetch the list of available packages + with requests.get(index_url) as response: + response.raise_for_status() + index = response.json() + + # generate urls and directories to download the packages + base_url = index_url.rsplit("/", 1)[0] + apk_dir = DEFAULT_WORK_DIR / arch / "apk" + apk_dir.mkdir(parents=True, exist_ok=True) + extract_dir = DEFAULT_WORK_DIR / arch / "extracted" + extract_dir.mkdir(parents=True, exist_ok=True) + dist_dir = DEFAULT_DIST_DIR / makefile_arch + dist_dir.mkdir(parents=True, exist_ok=True) + + # download only the used packages + for package, pkg_version in index.get("packages", {}).items(): + if package not in PACKAGES: + continue + filename = f"{package}-{pkg_version}.apk" + download_file(f"{base_url}/{filename}", apk_dir / filename) + extract_apk(apk_dir / filename, extract_dir) + collect_targets(package, extract_dir, dist_dir, makefile_arch, entries) + + missing = [suffix for suffix in DIST_SUFFIX_ORDER if len(entries.get(suffix, {})) != len(makefile_arches)] + if missing: + fail(f"missing entries for {', '.join(missing)}, check the upstream indexes") + + warn_on_diverging_filenames(entries, makefile_arches) + write_dist_mk(entries, makefile_arches) + + print(f"Upload tree ready in {DEFAULT_DIST_DIR}, publish it manually to the netifyd-dist mirror") + + +def main() -> None: + args = parse_args() + run_update(args) + + +if __name__ == "__main__": + main()